User Inputs

output.var = params$output.var 

transform.abs = FALSE
log.pred = params$log.pred
norm.pred = FALSE
algo.forward.caret = params$algo.forward.caret
algo.backward.caret = params$algo.backward.caret
algo.stepwise.caret = params$algo.stepwise.caret
algo.LASSO.caret = params$algo.LASSO.caret
algo.LARS.caret = params$algo.LARS.caret
message("Parameters used for training/prediction: ")
## Parameters used for training/prediction:
str(params)
## List of 7
##  $ output.var         : chr "y3"
##  $ log.pred           : logi TRUE
##  $ algo.forward.caret : logi TRUE
##  $ algo.backward.caret: logi TRUE
##  $ algo.stepwise.caret: logi TRUE
##  $ algo.LASSO.caret   : logi TRUE
##  $ algo.LARS.caret    : logi TRUE
# Setup Labels
output.var.tr = if (log.pred == TRUE)  paste0(output.var,'.log') else  output.var.tr = output.var

Loading Data

feat  = read.csv('../../Data/features_highprec.csv')
labels = read.csv('../../Data/labels.csv')
predictors = names(dplyr::select(feat,-JobName))
data.ori = inner_join(feat,labels,by='JobName')
#data.ori = inner_join(feat,select_at(labels,c('JobName',output.var)),by='JobName')

Data validation

cc  = complete.cases(data.ori)
data.notComplete = data.ori[! cc,]
data = data.ori[cc,] %>% select_at(c(predictors,output.var,'JobName'))
message('Original cases: ',nrow(data.ori))
## Original cases: 10000
message('Non-Complete cases: ',nrow(data.notComplete))
## Non-Complete cases: 3020
message('Complete cases: ',nrow(data))
## Complete cases: 6980
summary(dplyr::select_at(data,c('JobName',output.var)))
##       JobName           y3        
##  Job_00001:   1   Min.   : 95.91  
##  Job_00002:   1   1st Qu.:118.29  
##  Job_00003:   1   Median :124.03  
##  Job_00004:   1   Mean   :125.40  
##  Job_00007:   1   3rd Qu.:131.06  
##  Job_00008:   1   Max.   :193.73  
##  (Other)  :6974

Output Variable

The Output Variable y3 shows right skewness, so will proceed with a log transformation

Histogram

df=gather(select_at(data,output.var))
ggplot(df, aes(x=value)) + 
  geom_histogram(aes(y=..density..),bins = 50,fill='light blue') + 
  geom_density() 

  #stat_function(fun = dnorm, n = 100, args = list(mean = mean(df$value), sd = sd(df$value)))  

QQPlot

ggplot(gather(select_at(data,output.var)), aes(sample=value)) + 
  stat_qq() + 
  facet_wrap(~key, scales = 'free',ncol=4)

Trasformation of Output Variable from y3 to y3.log

if(log.pred==TRUE) data[[output.var.tr]] = log(data[[output.var]],10) else
  data[[output.var.tr]] = data[[output.var]]
df=gather(select_at(data,c(output.var,output.var.tr)))
ggplot(df, aes(value)) + 
  geom_histogram(aes(y=..density..),bins = 50,fill='light blue') + 
  geom_density() + 
  # stat_function(fun = dnorm, n = 100, args = list(mean = mean(df$value), sd = sd(df$value)))  
  facet_wrap(~key, scales = 'free',ncol=2)

ggplot(gather(select_at(data,c(output.var,output.var.tr))), aes(sample=value)) + 
  stat_qq() + 
  facet_wrap(~key, scales = 'free',ncol=4)

Best Normalizator y3

Normalization of y3 using bestNormalize package. (suggested orderNorm) This is cool, but I think is too far for the objective of the project

t=bestNormalize::bestNormalize(data[[output.var]])
t
## Best Normalizing transformation with 6980 Observations
##  Estimated Normality Statistics (Pearson P / df, lower => more normal):
##  - No transform: 2.9541 
##  - Box-Cox: 1.4767 
##  - Log_b(x+a): 2.0465 
##  - sqrt(x+a): 2.4458 
##  - exp(x): 749.7236 
##  - arcsinh(x): 2.0465 
##  - Yeo-Johnson: 1.2399 
##  - orderNorm: 1.1879 
## Estimation method: Out-of-sample via CV with 10 folds and 5 repeats
##  
## Based off these, bestNormalize chose:
## orderNorm Transformation with 6980 nonmissing obs and no ties 
##  - Original quantiles:
##      0%     25%     50%     75%    100% 
##  95.913 118.289 124.030 131.059 193.726
qqnorm(data[[output.var]])

qqnorm(predict(t))

orderNorm() is a rank-based procedure by which the values of a vector are mapped to their percentile, which is then mapped to the same percentile of the normal distribution. Without the presence of ties, this essentially guarantees that the transformation leads to a uniform distribution

Predictors

All predictors show a Fat-Tail situation, where the two tails are very tall, and a low distribution around the mean. The orderNorm transformation can help (see [Best Normalizator] section)

Interesting Predictors

Histograms

cols = c('x11','x18','stat98','x7','stat110')
df=gather(select_at(data,cols))
ggplot(df, aes(value)) + 
  geom_histogram(aes(y=..density..),bins = 50,fill='light blue') + 
  geom_density() + 
  # stat_function(fun = dnorm, n = 100, args = list(mean = mean(df$value), sd = sd(df$value)))  
  facet_wrap(~key, scales = 'free',ncol=3)

# ggplot(gather(select_at(data,cols)), aes(sample=value)) + 
#   stat_qq()+
#   facet_wrap(~key, scales = 'free',ncol=2)

lapply(select_at(data,cols),summary)
## $x11
##      Min.   1st Qu.    Median      Mean   3rd Qu.      Max. 
## 9.000e-08 9.494e-08 1.001e-07 1.001e-07 1.052e-07 1.100e-07 
## 
## $x18
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   1.500   3.147   4.769   4.772   6.418   7.999 
## 
## $stat98
##      Min.   1st Qu.    Median      Mean   3rd Qu.      Max. 
## -2.998619 -1.551882 -0.015993 -0.005946  1.528405  2.999499 
## 
## $x7
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   0.700   1.266   1.854   1.852   2.446   3.000 
## 
## $stat110
##      Min.   1st Qu.    Median      Mean   3rd Qu.      Max. 
## -2.999543 -1.496865 -0.002193 -0.004129  1.504273  2.999563

Scatter plot vs. output variable **y3.log

d = gather(dplyr::select_at(data,c(cols,output.var.tr)),key=target,value=value,-!!output.var.tr)
ggplot(data=d, aes_string(x='value',y=output.var.tr)) + 
  geom_point(color='light green',alpha=0.5) + 
  geom_smooth() + 
  facet_wrap(~target, scales = 'free',ncol=3)
## `geom_smooth()` using method = 'gam' and formula 'y ~ s(x, bs = "cs")'

All Predictors

Histograms

All indicators have a strong indication of Fat-Tails

df=gather(select_at(data,predictors))
ggplot(df, aes(value)) + 
  geom_histogram(aes(y=..density..),bins = 50,fill='light blue') + 
  geom_density() + 
  # stat_function(fun = dnorm, n = 100, args = list(mean = mean(df$value), sd = sd(df$value)))  
  facet_wrap(~key, scales = 'free',ncol=4)

Correlations

With Output Variable

#chart.Correlation(select(data,-JobName),  pch=21)
t=as.data.frame(round(cor(dplyr::select(data,-one_of(output.var.tr,'JobName'))
                          ,select_at(data,output.var.tr)),4))  %>%
  rownames_to_column(var='variable') %>% filter(variable != !!output.var) %>% arrange(-y3.log)
#DT::datatable(t)
message("Top Positive")
## Top Positive
kable(head(arrange(t,desc(y3.log)),20))
variable y3.log
x18 0.3120
x7 0.2091
stat98 0.1784
x9 0.1127
x17 0.0611
x16 0.0489
x10 0.0472
x21 0.0412
x11 0.0322
x8 0.0318
stat156 0.0287
stat23 0.0234
stat100 0.0206
stat144 0.0203
stat59 0.0202
stat60 0.0199
stat195 0.0199
stat141 0.0194
stat73 0.0192
stat197 0.0185
message("Top Negative")
## Top Negative
kable(head(arrange(t,y3.log),20))
variable y3.log
stat110 -0.1594
x4 -0.0603
stat13 -0.0345
stat41 -0.0345
stat14 -0.0317
stat149 -0.0309
stat113 -0.0279
stat4 -0.0248
stat106 -0.0236
stat146 -0.0236
stat186 -0.0217
stat91 -0.0210
stat214 -0.0209
stat5 -0.0207
stat22 -0.0202
stat39 -0.0202
stat175 -0.0194
stat187 -0.0193
stat128 -0.0192
stat37 -0.0191

Between All Variables

#chart.Correlation(select(data,-JobName),  pch=21)
t=as.data.frame(round(cor(dplyr::select(data,-one_of('JobName'))),4))
#DT::datatable(t,options=list(scrollX=T))
message("Showing only 10 variables")
## Showing only 10 variables
kable(t[1:10,1:10])
x1 x2 x3 x4 x5 x6 x7 x8 x9 x10
x1 1.0000 0.0034 -0.0028 0.0085 0.0068 0.0159 0.0264 -0.0012 0.0142 0.0013
x2 0.0034 1.0000 -0.0057 0.0004 -0.0094 -0.0101 0.0089 0.0078 0.0049 -0.0214
x3 -0.0028 -0.0057 1.0000 0.0029 0.0046 0.0006 -0.0105 -0.0002 0.0167 -0.0137
x4 0.0085 0.0004 0.0029 1.0000 -0.0059 0.0104 0.0098 0.0053 0.0061 -0.0023
x5 0.0068 -0.0094 0.0046 -0.0059 1.0000 0.0016 -0.0027 0.0081 0.0259 -0.0081
x6 0.0159 -0.0101 0.0006 0.0104 0.0016 1.0000 0.0200 -0.0157 0.0117 -0.0072
x7 0.0264 0.0089 -0.0105 0.0098 -0.0027 0.0200 1.0000 -0.0018 -0.0069 -0.0221
x8 -0.0012 0.0078 -0.0002 0.0053 0.0081 -0.0157 -0.0018 1.0000 0.0142 -0.0004
x9 0.0142 0.0049 0.0167 0.0061 0.0259 0.0117 -0.0069 0.0142 1.0000 0.0149
x10 0.0013 -0.0214 -0.0137 -0.0023 -0.0081 -0.0072 -0.0221 -0.0004 0.0149 1.0000

Scatter Plots with Output Variable

Scatter plots with all predictors and the output variable (y3.log)

d = gather(dplyr::select_at(data,c(predictors,output.var.tr)),key=target,value=value,-!!output.var.tr)
ggplot(data=d, aes_string(x='value',y=output.var.tr)) + 
  geom_point(color='light blue',alpha=0.5) + 
  geom_smooth() + 
  facet_wrap(~target, scales = 'free',ncol=4)
## `geom_smooth()` using method = 'gam' and formula 'y ~ s(x, bs = "cs")'

Multicollinearity - VIF

No Multicollinearity among predictors

Showing Top predictor by VIF Value

vifDF = usdm::vif(select_at(data,predictors)) %>% arrange(desc(VIF))
head(vifDF,15)
##    Variables      VIF
## 1    stat202 1.063592
## 2    stat141 1.062435
## 3     stat52 1.062123
## 4    stat178 1.062030
## 5    stat164 1.059900
## 6    stat184 1.059400
## 7     stat70 1.058888
## 8    stat150 1.058825
## 9     stat14 1.058728
## 10    stat37 1.058385
## 11   stat147 1.058213
## 12   stat113 1.057934
## 13   stat209 1.057891
## 14       x14 1.057831
## 15   stat166 1.057729

Feature Eng

  • Square Root transformation for x18
data.tr=data %>%
  mutate(x18.sqrt = sqrt(x18)) 
cols=c('x18','x18.sqrt')

Comparing Pre and Post Transformation Density Plots

# ggplot(gather(select_at(data.tr,cols)), aes(value)) + 
#   geom_histogram(aes(y=..density..),bins = 50,fill='light blue') + 
#   geom_density() + 
#   facet_wrap(~key, scales = 'free',ncol=4)

d = gather(dplyr::select_at(data.tr,c(cols,output.var.tr)),key=target,value=value,-!!output.var.tr)
ggplot(data=d, aes_string(x='value',y=output.var.tr)) + 
  geom_point(color='light blue',alpha=0.5) + 
  geom_smooth() + 
  facet_wrap(~target, scales = 'free',ncol=4)
## `geom_smooth()` using method = 'gam' and formula 'y ~ s(x, bs = "cs")'

#removing unwanted variables
data.tr=data.tr %>%
  dplyr::select_at(names(data.tr)[! names(data.tr) %in% c('x18','y3','JobName')])

data=data.tr
label.names=output.var.tr

Modeling

Train Test Split

data = data[sample(nrow(data)),] # randomly shuffle data
split = sample.split(data[,label.names], SplitRatio = 0.8)

data.train = subset(data, split == TRUE)
data.test = subset(data, split == FALSE)

Common Functions

plot.diagnostics <-  function(model, train) {
  plot(model)
  
  residuals = resid(model) # Plotted above in plot(lm.out)
  r.standard = rstandard(model)
  r.student = rstudent(model)
  
  df = data.frame(x=predict(model,train),y=r.student)
  p=ggplot(data=df,aes(x=x,y=y)) +
    geom_point(color='blue',alpha=0.5,shape=20,size=2) +
    geom_hline(yintercept = 0,size=1)+
    ylab("Student Residuals") +
    xlab("Predicted Values")+
    ggtitle("Student Residual Plot")
  plot(p)
  
  df = data.frame(x=predict(model,train),y=r.standard)
  p=ggplot(data=df,aes(x=x,y=y)) +
    geom_point(color='blue',alpha=0.5,shape=20,size=2) +
    geom_hline(yintercept = c(-2,0,2),size=1)+
    ylab("Student Residuals") +
    xlab("Predicted Values")+
    ggtitle("Student Residual Plot")
  plot(p)
  # Histogram
  df=data.frame(r.student)
  p=ggplot(data=df,aes(r.student)) +
    geom_histogram(aes(y=..density..),bins = 50,fill='blue',alpha=0.6) + 
    stat_function(fun = dnorm, n = 100, args = list(mean = 0, sd = 1)) +
    ylab("Density")+
    xlab("Studentized Residuals")+
    ggtitle("Distribution of Studentized Residuals")
  plot(p)
  # http://www.stat.columbia.edu/~martin/W2024/R7.pdf
  # Influential plots
  inf.meas = influence.measures(model)
  # print (summary(inf.meas)) # too much data
  
  # Leverage plot
  lev = hat(model.matrix(model))
  df=tibble::rownames_to_column(as.data.frame(lev),'id')
  p=ggplot(data=df,aes(x=as.numeric(id),y=lev)) +
    geom_point(color='blue',alpha=0.5,shape=20,size=2) +
    ylab('Leverage - check') + 
    xlab('Index')
  plot(p)
  # Cook's Distance
  cd = cooks.distance(model)
  df=tibble::rownames_to_column(as.data.frame(cd),'id')
  p=ggplot(data=df,aes(x=as.numeric(id),y=cd)) +
    geom_point(color='blue',alpha=0.5,shape=20,size=2) +
    geom_text(data=filter(df,cd>15/nrow(train)),aes(label=id),check_overlap=T,size=3,vjust=-.5)+
    ylab('Cooks distances') + 
    geom_hline(yintercept = c(4/nrow(train),0),size=1)+
    xlab('Index')
  plot(p)
  print (paste("Number of data points that have Cook's D > 4/n: ", length(cd[cd > 4/nrow(train)]), sep = "")) 
  print (paste("Number of data points that have Cook's D > 1: ", length(cd[cd > 1]), sep = "")) 
  return(cd)
}

# function to set up random seeds
# Based on http://jaehyeon-kim.github.io/2015/05/Setup-Random-Seeds-on-Caret-Package.html 
setCaretSeeds <- function(method = "cv", numbers = 1, repeats = 1, tunes = NULL, seed = 1701) {
  #B is the number of resamples and integer vector of M (numbers + tune length if any)
  B <- if (method == "cv") numbers
  else if(method == "repeatedcv") numbers * repeats
  else NULL
  if(is.null(length)) {
    seeds <- NULL
  } else {
    set.seed(seed = seed)
    seeds <- vector(mode = "list", length = B)
    seeds <- lapply(seeds, function(x) sample.int(n = 1000000
                                                  , size = numbers + ifelse(is.null(tunes), 0, tunes)))
    seeds[[length(seeds) + 1]] <- sample.int(n = 1000000, size = 1)
  }
  # return seeds
  seeds
}

train.caret.glmselect = function(formula, data, method
                                 ,subopt = NULL, feature.names
                                 , train.control = NULL, tune.grid = NULL, pre.proc = NULL){
  
  if(is.null(train.control)){
    train.control <- trainControl(method = "cv"
                              ,number = 10
                              ,seeds = setCaretSeeds(method = "cv"
                                                     , numbers = 10
                                                     , seed = 1701)
                              ,search = "grid"
                              ,verboseIter = TRUE
                              ,allowParallel = TRUE
                              )
  }
  
  if(is.null(tune.grid)){
    if (method == 'leapForward' | method == 'leapBackward' | method == 'leapSeq'){
      tune.grid = data.frame(nvmax = 1:length(feature.names))
    }
    if (method == 'glmnet' && subopt == 'LASSO'){
      # Will only show 1 Lambda value during training, but that is OK
      # https://stackoverflow.com/questions/47526544/why-need-to-tune-lambda-with-carettrain-method-glmnet-and-cv-glmnet
      # Another option for LASSO is this: https://github.com/topepo/caret/blob/master/RegressionTests/Code/lasso.R
      lambda = 10^seq(-2,0, length =100)
      alpha = c(1)
      tune.grid = expand.grid(alpha = alpha,lambda = lambda)
    }
    if (method == 'lars'){
      # https://github.com/topepo/caret/blob/master/RegressionTests/Code/lars.R
      fraction = seq(0, 1, length = 100)
      tune.grid = expand.grid(fraction = fraction)
      pre.proc = c("center", "scale") 
    }
  }
  
  # http://sshaikh.org/2015/05/06/parallelize-machine-learning-in-r-with-multi-core-cpus/
  cl <- makeCluster(ceiling(detectCores()*0.85)) # use 75% of cores only, leave rest for other tasks
  registerDoParallel(cl)

  set.seed(1) 
  # note that the seed has to actually be set just before this function is called
  # settign is above just not ensure reproducibility for some reason
  model.caret <- caret::train(formula
                              , data = data
                              , method = method
                              , tuneGrid = tune.grid
                              , trControl = train.control
                              , preProc = pre.proc
                              )
  
  stopCluster(cl)
  registerDoSEQ() # register sequential engine in case you are not using this function anymore
  
  if (method == 'leapForward' | method == 'leapBackward' | method == 'leapSeq'){
    print("All models results")
    print(model.caret$results) # all model results
    print("Best Model")
    print(model.caret$bestTune) # best model
    model = model.caret$finalModel

    # Metrics Plot 
    dataPlot = model.caret$results %>%
      gather(key='metric',value='value',-nvmax) %>%
      dplyr::filter(metric %in% c('MAE','RMSE','Rsquared'))
    metricsPlot = ggplot(data=dataPlot,aes(x=nvmax,y=value) ) +
      geom_line(color='lightblue4') +
      geom_point(color='blue',alpha=0.7,size=.9) +
      facet_wrap(~metric,ncol=2,scales='free_y')+
      theme_light()
    plot(metricsPlot)
    
    # Residuals Plot
    # leap function does not support studentized residuals
    dataPlot=data.frame(pred=predict(model.caret,data),res=resid(model.caret))
    residPlot = ggplot(dataPlot,aes(x=pred,y=res)) +
      geom_point(color='light blue',alpha=0.7) +
      geom_smooth(method="lm")+
      theme_light()
    plot(residPlot)
   
    residHistogram = ggplot(dataPlot,aes(x=res)) +
      geom_histogram(aes(y=..density..),fill='light blue',alpha=1) +
      #geom_density(color='lightblue4') + 
      stat_function(fun = dnorm, n = 100, args = list(mean = mean(dataPlot$res)
                                                       , sd = sd(dataPlot$res)),color='lightblue4')  
      theme_light()
    plot(residHistogram)
    id = rownames(model.caret$bestTune)    
    # Provides the coefficients of the best model
    # regsubsets doens return a full model (see documentation of regsubset), so we need to recalcualte themodel
    # https://stackoverflow.com/questions/13063762/how-to-obtain-a-lm-object-from-regsubsets
    print("Coefficients of final model:")
    coefs <- coef(model, id=id)
    #calculate the model to the the coef intervals
    nams <- names(coefs)
    nams <- nams[!nams %in% "(Intercept)"]
    response <-  as.character(formula[[2]])
    form <- as.formula(paste(response, paste(nams, collapse = " + "), sep = " ~ "))
    mod <- lm(form, data = data)
    #coefs
    #coef(mod)
    print(car::Confint(mod))
    return(list(model = model,id = id, residPlot = residPlot, residHistogram=residHistogram
                ,modelLM=mod))
  }
  if (method == 'glmnet' && subopt == 'LASSO'){
    print(model.caret)
    print(plot(model.caret))
    print(model.caret$bestTune)
    
    print(model.caret$results)
    model=model.caret$finalModel
    # Metrics Plot 
    dataPlot = model.caret$results %>%
      gather(key='metric',value='value',-lambda) %>%
      dplyr::filter(metric %in% c('MAE','RMSE','Rsquared'))
    metricsPlot = ggplot(data=dataPlot,aes(x=lambda,y=value) ) +
      geom_line(color='lightblue4') +
      geom_point(color='blue',alpha=0.7,size=.9) +
      facet_wrap(~metric,ncol=2,scales='free_y')+
      theme_light()
    plot(metricsPlot)
    
    # Residuals Plot 
    dataPlot=data.frame(pred=predict(model.caret,data),res=resid(model.caret))
    residPlot = ggplot(dataPlot,aes(x=pred,y=res)) +
      geom_point(color='light blue',alpha=0.7) +
      geom_smooth(method="lm")+
      theme_light()
    plot(residPlot)

    residHistogram = ggplot(dataPlot,aes(x=res)) +
      geom_histogram(aes(y=..density..),fill='light blue',alpha=1) +
      #geom_density(color='lightblue4') +
      stat_function(fun = dnorm, n = 100, args = list(mean = mean(dataPlot$res)
                                                       , sd = sd(dataPlot$res)),color='lightblue4')  
      theme_light()
    plot(residHistogram)
    
    print("Coefficients") 
    #no interval for glmnet: https://stackoverflow.com/questions/39750965/confidence-intervals-for-ridge-regression
    t=coef(model,s=model.caret$bestTune$lambda)
    model.coef = t[which(t[,1]!=0),]
    print(as.data.frame(model.coef))
    id = NULL # not really needed but added for consistency
    return(list(model = model.caret,id = id, residPlot = residPlot, metricsPlot=metricsPlot ))
  }
  if (method == 'lars'){
    print(model.caret)
    print(plot(model.caret))
    print(model.caret$bestTune)
    
    # Metrics Plot
    dataPlot = model.caret$results %>%
        gather(key='metric',value='value',-fraction) %>%
      dplyr::filter(metric %in% c('MAE','RMSE','Rsquared'))
    metricsPlot = ggplot(data=dataPlot,aes(x=fraction,y=value) ) +
      geom_line(color='lightblue4') +
      geom_point(color='blue',alpha=0.7,size=.9) +
      facet_wrap(~metric,ncol=2,scales='free_y')+
      theme_light()
    plot(metricsPlot)
    
    # Residuals Plot
    dataPlot=data.frame(pred=predict(model.caret,data),res=resid(model.caret))
    residPlot = ggplot(dataPlot,aes(x=pred,y=res)) +
      geom_point(color='light blue',alpha=0.7) +
      geom_smooth(method="lm")+
      theme_light()
    plot(residPlot)

    residHistogram = ggplot(dataPlot,aes(x=res)) +
      geom_histogram(aes(y=..density..),fill='light blue',alpha=1) +
      #geom_density(color='lightblue4') + 
      stat_function(fun = dnorm, n = 100, args = list(mean = mean(dataPlot$res)
                                                       , sd = sd(dataPlot$res)),color='lightblue4')  
      theme_light()
    plot(residHistogram)
    
    print("Coefficients") 
    t=coef(model.caret$finalModel,s=model.caret$bestTune$fraction,mode='fraction')
    model.coef = t[which(t!=0)]
    print(model.coef)
    id = NULL # not really needed but added for consistency
    return(list(model = model.caret,id = id, residPlot = residPlot, residHistogram=residHistogram))
  }
}

# https://stackoverflow.com/questions/48265743/linear-model-subset-selection-goodness-of-fit-with-k-fold-cross-validation
# changed slightly since call[[2]] was just returning "formula" without actually returnign the value in formula
predict.regsubsets <- function(object, newdata, id, formula, ...) {
    #form <- as.formula(object$call[[2]])
    mat <- model.matrix(formula, newdata) # adds intercept and expands any interaction terms
    coefi <- coef(object, id = id)
    xvars <- names(coefi)
    return(mat[,xvars]%*%coefi)
}
  
test.model = function(model, test, level=0.95
                      ,draw.limits = FALSE, good = 0.1, ok = 0.15
                      ,method = NULL, subopt = NULL
                      ,id = NULL, formula, feature.names, label.names
                      ,transformation = NULL){
  ## if using caret for glm select equivalent functionality, 
  ## need to pass formula (full is ok as it will select subset of variables from there)
  if (is.null(method)){
    pred = predict(model, newdata=test, interval="confidence", level = level) 
  }
  
  if (method == 'leapForward' | method == 'leapBackward' | method == 'leapSeq'){
    pred = predict.regsubsets(model, newdata = test, id = id, formula = formula)
  }
  
  if (method == 'glmnet' && subopt == 'LASSO'){
    xtest = as.matrix(test[,feature.names]) 
    pred=as.data.frame(predict(model, xtest))
  }
  
  if (method == 'lars'){
    pred=as.data.frame(predict(model, newdata = test))
  }
    
  # Summary of predicted values
  print ("Summary of predicted values: ")
  print(summary(pred[,1]))

  test.mse = mean((test[,label.names]-pred[,1])^2)
  print (paste(method, subopt, "Test MSE:", test.mse, sep=" "))
  
  if(log.pred == TRUE || norm.pred == TRUE){
    # plot transformewd comparison first
    df=data.frame(x=test[,label.names],y=pred[,1])
    ggplot(df,aes(x=x,y=y)) +
      geom_point(color='blue',alpha=0.5,shape=20,size=2) +
      geom_abline(slope=1,intercept=0,color='black',size=1) +
      #scale_y_continuous(limits=c(min(df),max(df)))+
      xlab("Actual (Transformed)")+
      ylab("Predicted (Transformed)")
  }
    
  if (log.pred == FALSE && norm.pred == FALSE){
    x = test[,label.names]
    y = pred[,1]
  }
  if (log.pred == TRUE){
    x = 10^test[,label.names]
    y = 10^pred[,1]  
  }
  if (norm.pred == TRUE){
    x = predict(transformation, test[,label.names], inverse = TRUE)
    y = predict(transformation, pred[,1], inverse = TRUE)
  }

  df=data.frame(x,y)
  ggplot(df,aes(x,y)) +
    geom_point(color='blue',alpha=0.5,shape=20,size=2) +
    geom_abline(slope=c(1+good,1-good,1+ok,1-ok)
                ,intercept=rep(0,4),color=c('dark green','dark green','dark red','dark red'),size=1,alpha=0.8) +
    #scale_y_continuous(limits=c(min(df),max(df)))+
    xlab("Actual")+
    ylab("Predicted") 
    
 
}

Setup Formulae

n <- names(data.train)
 formula <- as.formula(paste(paste(n[n %in% label.names], collapse = " + ")
                             ," ~", paste(n[!n %in% label.names], collapse = " + "))) 

grand.mean.formula = as.formula(paste(paste(n[n %in% label.names], collapse = " + ")," ~ 1"))

print(formula)
## y3.log ~ x1 + x2 + x3 + x4 + x5 + x6 + x7 + x8 + x9 + x10 + x11 + 
##     x12 + x13 + x14 + x15 + x16 + x17 + x19 + x20 + x21 + x22 + 
##     x23 + stat1 + stat2 + stat3 + stat4 + stat5 + stat6 + stat7 + 
##     stat8 + stat9 + stat10 + stat11 + stat12 + stat13 + stat14 + 
##     stat15 + stat16 + stat17 + stat18 + stat19 + stat20 + stat21 + 
##     stat22 + stat23 + stat24 + stat25 + stat26 + stat27 + stat28 + 
##     stat29 + stat30 + stat31 + stat32 + stat33 + stat34 + stat35 + 
##     stat36 + stat37 + stat38 + stat39 + stat40 + stat41 + stat42 + 
##     stat43 + stat44 + stat45 + stat46 + stat47 + stat48 + stat49 + 
##     stat50 + stat51 + stat52 + stat53 + stat54 + stat55 + stat56 + 
##     stat57 + stat58 + stat59 + stat60 + stat61 + stat62 + stat63 + 
##     stat64 + stat65 + stat66 + stat67 + stat68 + stat69 + stat70 + 
##     stat71 + stat72 + stat73 + stat74 + stat75 + stat76 + stat77 + 
##     stat78 + stat79 + stat80 + stat81 + stat82 + stat83 + stat84 + 
##     stat85 + stat86 + stat87 + stat88 + stat89 + stat90 + stat91 + 
##     stat92 + stat93 + stat94 + stat95 + stat96 + stat97 + stat98 + 
##     stat99 + stat100 + stat101 + stat102 + stat103 + stat104 + 
##     stat105 + stat106 + stat107 + stat108 + stat109 + stat110 + 
##     stat111 + stat112 + stat113 + stat114 + stat115 + stat116 + 
##     stat117 + stat118 + stat119 + stat120 + stat121 + stat122 + 
##     stat123 + stat124 + stat125 + stat126 + stat127 + stat128 + 
##     stat129 + stat130 + stat131 + stat132 + stat133 + stat134 + 
##     stat135 + stat136 + stat137 + stat138 + stat139 + stat140 + 
##     stat141 + stat142 + stat143 + stat144 + stat145 + stat146 + 
##     stat147 + stat148 + stat149 + stat150 + stat151 + stat152 + 
##     stat153 + stat154 + stat155 + stat156 + stat157 + stat158 + 
##     stat159 + stat160 + stat161 + stat162 + stat163 + stat164 + 
##     stat165 + stat166 + stat167 + stat168 + stat169 + stat170 + 
##     stat171 + stat172 + stat173 + stat174 + stat175 + stat176 + 
##     stat177 + stat178 + stat179 + stat180 + stat181 + stat182 + 
##     stat183 + stat184 + stat185 + stat186 + stat187 + stat188 + 
##     stat189 + stat190 + stat191 + stat192 + stat193 + stat194 + 
##     stat195 + stat196 + stat197 + stat198 + stat199 + stat200 + 
##     stat201 + stat202 + stat203 + stat204 + stat205 + stat206 + 
##     stat207 + stat208 + stat209 + stat210 + stat211 + stat212 + 
##     stat213 + stat214 + stat215 + stat216 + stat217 + x18.sqrt
print(grand.mean.formula)
## y3.log ~ 1
# Update feature.names because we may have transformed some features
feature.names = n[!n %in% label.names]

Full Model

model.full = lm(formula , data.train)
summary(model.full)
## 
## Call:
## lm(formula = formula, data = data.train)
## 
## Residuals:
##       Min        1Q    Median        3Q       Max 
## -0.083126 -0.020414 -0.004746  0.016240  0.188245 
## 
## Coefficients:
##               Estimate Std. Error t value Pr(>|t|)    
## (Intercept)  1.970e+00  9.597e-03 205.243  < 2e-16 ***
## x1          -5.744e-04  6.574e-04  -0.874 0.382238    
## x2           4.763e-04  4.160e-04   1.145 0.252323    
## x3           8.001e-05  1.146e-04   0.698 0.485126    
## x4          -5.252e-05  9.046e-06  -5.806 6.76e-09 ***
## x5           2.788e-04  2.955e-04   0.943 0.345488    
## x6           2.443e-04  5.966e-04   0.410 0.682185    
## x7           1.136e-02  6.408e-04  17.728  < 2e-16 ***
## x8           3.673e-04  1.488e-04   2.468 0.013599 *  
## x9           3.299e-03  3.309e-04   9.970  < 2e-16 ***
## x10          1.172e-03  3.082e-04   3.804 0.000144 ***
## x11          1.889e+05  7.379e+04   2.560 0.010499 *  
## x12         -4.195e-05  1.887e-04  -0.222 0.824042    
## x13          6.336e-05  7.490e-05   0.846 0.397684    
## x14         -4.126e-04  3.219e-04  -1.282 0.200018    
## x15         -7.729e-05  3.087e-04  -0.250 0.802320    
## x16          8.416e-04  2.150e-04   3.915 9.16e-05 ***
## x17          1.519e-03  3.254e-04   4.670 3.09e-06 ***
## x19          1.574e-04  1.663e-04   0.947 0.343782    
## x20         -8.774e-04  1.145e-03  -0.766 0.443526    
## x21          1.298e-04  4.216e-05   3.078 0.002093 ** 
## x22         -5.997e-04  3.457e-04  -1.735 0.082801 .  
## x23          5.539e-05  3.280e-04   0.169 0.865905    
## stat1       -1.555e-04  2.490e-04  -0.624 0.532328    
## stat2        2.173e-04  2.482e-04   0.875 0.381389    
## stat3        3.622e-04  2.488e-04   1.456 0.145573    
## stat4       -5.704e-04  2.485e-04  -2.295 0.021752 *  
## stat5       -2.720e-04  2.508e-04  -1.084 0.278234    
## stat6       -3.917e-04  2.511e-04  -1.560 0.118858    
## stat7       -1.234e-04  2.500e-04  -0.494 0.621623    
## stat8        1.699e-04  2.481e-04   0.685 0.493329    
## stat9        1.449e-04  2.479e-04   0.585 0.558711    
## stat10      -2.524e-04  2.486e-04  -1.015 0.310091    
## stat11      -2.595e-04  2.499e-04  -1.038 0.299210    
## stat12       9.192e-05  2.485e-04   0.370 0.711528    
## stat13      -6.822e-04  2.480e-04  -2.751 0.005964 ** 
## stat14      -7.839e-04  2.476e-04  -3.167 0.001551 ** 
## stat15      -2.587e-04  2.461e-04  -1.051 0.293087    
## stat16       1.189e-04  2.493e-04   0.477 0.633384    
## stat17       1.749e-07  2.463e-04   0.001 0.999433    
## stat18      -4.600e-04  2.477e-04  -1.857 0.063317 .  
## stat19       8.759e-05  2.468e-04   0.355 0.722656    
## stat20      -2.069e-04  2.480e-04  -0.835 0.404004    
## stat21       1.849e-05  2.488e-04   0.074 0.940786    
## stat22      -4.362e-04  2.499e-04  -1.745 0.080965 .  
## stat23       6.593e-04  2.487e-04   2.651 0.008048 ** 
## stat24      -4.303e-04  2.484e-04  -1.733 0.083206 .  
## stat25      -3.820e-04  2.482e-04  -1.539 0.123795    
## stat26      -3.497e-04  2.473e-04  -1.414 0.157414    
## stat27       7.609e-05  2.490e-04   0.306 0.759922    
## stat28      -1.258e-04  2.487e-04  -0.506 0.612889    
## stat29      -2.699e-05  2.482e-04  -0.109 0.913406    
## stat30       3.636e-04  2.497e-04   1.456 0.145397    
## stat31       3.007e-05  2.509e-04   0.120 0.904599    
## stat32       7.169e-06  2.512e-04   0.029 0.977236    
## stat33      -1.487e-04  2.471e-04  -0.602 0.547304    
## stat34       6.438e-05  2.468e-04   0.261 0.794200    
## stat35      -4.613e-04  2.496e-04  -1.848 0.064630 .  
## stat36      -7.883e-05  2.473e-04  -0.319 0.749883    
## stat37      -3.300e-04  2.499e-04  -1.320 0.186754    
## stat38       1.733e-04  2.488e-04   0.696 0.486319    
## stat39      -3.432e-04  2.469e-04  -1.390 0.164552    
## stat40      -7.324e-05  2.491e-04  -0.294 0.768757    
## stat41      -4.256e-04  2.476e-04  -1.719 0.085760 .  
## stat42      -1.520e-04  2.483e-04  -0.612 0.540596    
## stat43      -2.053e-04  2.500e-04  -0.821 0.411541    
## stat44      -2.094e-04  2.476e-04  -0.846 0.397695    
## stat45      -4.338e-04  2.479e-04  -1.750 0.080178 .  
## stat46       3.379e-04  2.483e-04   1.361 0.173611    
## stat47       1.710e-04  2.512e-04   0.680 0.496223    
## stat48       2.390e-04  2.483e-04   0.963 0.335822    
## stat49       1.063e-04  2.460e-04   0.432 0.665793    
## stat50       1.586e-04  2.463e-04   0.644 0.519791    
## stat51       4.864e-04  2.469e-04   1.970 0.048836 *  
## stat52      -3.060e-04  2.485e-04  -1.231 0.218211    
## stat53      -3.595e-04  2.506e-04  -1.434 0.151497    
## stat54      -4.010e-04  2.514e-04  -1.595 0.110774    
## stat55       1.200e-04  2.464e-04   0.487 0.626323    
## stat56      -3.103e-04  2.494e-04  -1.244 0.213628    
## stat57       1.007e-04  2.460e-04   0.409 0.682395    
## stat58       2.262e-04  2.477e-04   0.913 0.361229    
## stat59       4.114e-04  2.484e-04   1.656 0.097693 .  
## stat60       3.227e-04  2.495e-04   1.294 0.195877    
## stat61      -1.303e-04  2.498e-04  -0.521 0.602064    
## stat62      -1.818e-04  2.471e-04  -0.736 0.461975    
## stat63       2.458e-04  2.497e-04   0.984 0.324976    
## stat64      -1.720e-04  2.465e-04  -0.698 0.485495    
## stat65      -3.932e-04  2.484e-04  -1.583 0.113513    
## stat66       1.680e-04  2.505e-04   0.670 0.502600    
## stat67       9.367e-05  2.491e-04   0.376 0.706928    
## stat68      -2.975e-05  2.488e-04  -0.120 0.904841    
## stat69      -8.536e-05  2.493e-04  -0.342 0.732074    
## stat70       3.284e-04  2.474e-04   1.327 0.184413    
## stat71       2.651e-04  2.465e-04   1.075 0.282211    
## stat72       1.842e-04  2.496e-04   0.738 0.460609    
## stat73      -5.518e-05  2.498e-04  -0.221 0.825203    
## stat74      -1.484e-04  2.489e-04  -0.596 0.551033    
## stat75      -1.221e-04  2.497e-04  -0.489 0.624909    
## stat76       3.062e-04  2.494e-04   1.228 0.219623    
## stat77      -9.509e-05  2.478e-04  -0.384 0.701196    
## stat78      -2.082e-05  2.483e-04  -0.084 0.933180    
## stat79      -1.718e-04  2.501e-04  -0.687 0.492256    
## stat80       2.861e-04  2.497e-04   1.146 0.251876    
## stat81       4.374e-05  2.491e-04   0.176 0.860617    
## stat82       2.561e-04  2.473e-04   1.036 0.300394    
## stat83      -8.803e-05  2.490e-04  -0.354 0.723659    
## stat84      -3.412e-04  2.477e-04  -1.378 0.168370    
## stat85       3.441e-06  2.488e-04   0.014 0.988967    
## stat86       1.240e-04  2.494e-04   0.497 0.619004    
## stat87      -3.114e-04  2.495e-04  -1.248 0.212031    
## stat88      -1.330e-04  2.459e-04  -0.541 0.588741    
## stat89      -2.857e-05  2.466e-04  -0.116 0.907766    
## stat90      -1.037e-04  2.503e-04  -0.414 0.678656    
## stat91      -4.426e-04  2.461e-04  -1.798 0.072172 .  
## stat92      -2.730e-04  2.475e-04  -1.103 0.270111    
## stat93      -3.281e-04  2.503e-04  -1.311 0.189983    
## stat94      -3.295e-04  2.504e-04  -1.316 0.188218    
## stat95      -2.181e-05  2.487e-04  -0.088 0.930132    
## stat96      -2.387e-04  2.473e-04  -0.965 0.334447    
## stat97       2.776e-04  2.463e-04   1.127 0.259788    
## stat98       3.599e-03  2.455e-04  14.659  < 2e-16 ***
## stat99       3.685e-04  2.499e-04   1.475 0.140399    
## stat100      5.062e-04  2.498e-04   2.027 0.042727 *  
## stat101     -3.069e-04  2.509e-04  -1.223 0.221329    
## stat102     -1.575e-05  2.499e-04  -0.063 0.949761    
## stat103     -4.194e-04  2.506e-04  -1.673 0.094353 .  
## stat104     -2.083e-04  2.477e-04  -0.841 0.400386    
## stat105      2.701e-04  2.472e-04   1.093 0.274523    
## stat106     -3.922e-04  2.473e-04  -1.586 0.112726    
## stat107     -2.229e-05  2.472e-04  -0.090 0.928140    
## stat108     -2.580e-04  2.476e-04  -1.042 0.297355    
## stat109     -7.882e-05  2.486e-04  -0.317 0.751176    
## stat110     -3.282e-03  2.473e-04 -13.273  < 2e-16 ***
## stat111     -1.768e-04  2.463e-04  -0.718 0.472955    
## stat112     -7.420e-05  2.501e-04  -0.297 0.766742    
## stat113     -4.284e-04  2.510e-04  -1.707 0.087947 .  
## stat114      6.271e-05  2.483e-04   0.253 0.800588    
## stat115      3.301e-04  2.478e-04   1.332 0.182959    
## stat116      2.888e-04  2.498e-04   1.156 0.247724    
## stat117      1.812e-04  2.499e-04   0.725 0.468257    
## stat118     -1.041e-04  2.461e-04  -0.423 0.672270    
## stat119     -1.341e-05  2.471e-04  -0.054 0.956711    
## stat120      1.633e-04  2.465e-04   0.663 0.507630    
## stat121     -2.475e-04  2.485e-04  -0.996 0.319197    
## stat122      4.396e-05  2.472e-04   0.178 0.858893    
## stat123      1.133e-04  2.513e-04   0.451 0.652025    
## stat124     -2.406e-04  2.476e-04  -0.972 0.331116    
## stat125      2.617e-04  2.480e-04   1.055 0.291376    
## stat126      2.880e-05  2.472e-04   0.117 0.907247    
## stat127     -1.404e-04  2.484e-04  -0.565 0.571963    
## stat128     -1.738e-04  2.485e-04  -0.699 0.484345    
## stat129      1.268e-04  2.480e-04   0.511 0.609362    
## stat130      1.799e-04  2.477e-04   0.726 0.467748    
## stat131      2.904e-04  2.490e-04   1.166 0.243568    
## stat132      6.262e-06  2.473e-04   0.025 0.979802    
## stat133      7.600e-06  2.489e-04   0.031 0.975648    
## stat134     -1.866e-04  2.463e-04  -0.758 0.448582    
## stat135     -7.078e-05  2.478e-04  -0.286 0.775174    
## stat136      3.268e-05  2.497e-04   0.131 0.895881    
## stat137      1.627e-04  2.473e-04   0.658 0.510594    
## stat138      9.145e-05  2.476e-04   0.369 0.711872    
## stat139      9.221e-05  2.493e-04   0.370 0.711503    
## stat140      2.189e-04  2.463e-04   0.889 0.374094    
## stat141      7.575e-05  2.461e-04   0.308 0.758228    
## stat142      5.752e-05  2.491e-04   0.231 0.817358    
## stat143      2.554e-04  2.476e-04   1.032 0.302277    
## stat144      2.672e-04  2.474e-04   1.080 0.280190    
## stat145     -6.224e-05  2.517e-04  -0.247 0.804745    
## stat146     -4.353e-04  2.499e-04  -1.742 0.081524 .  
## stat147     -2.299e-04  2.503e-04  -0.918 0.358520    
## stat148     -3.718e-04  2.463e-04  -1.510 0.131184    
## stat149     -4.619e-04  2.493e-04  -1.853 0.063961 .  
## stat150      1.418e-04  2.486e-04   0.570 0.568400    
## stat151      6.500e-05  2.517e-04   0.258 0.796278    
## stat152     -1.158e-04  2.462e-04  -0.470 0.638219    
## stat153      2.442e-05  2.524e-04   0.097 0.922941    
## stat154      1.013e-04  2.508e-04   0.404 0.686283    
## stat155     -2.949e-04  2.472e-04  -1.193 0.232868    
## stat156      3.909e-04  2.504e-04   1.561 0.118606    
## stat157     -8.436e-05  2.464e-04  -0.342 0.732136    
## stat158     -3.429e-04  2.509e-04  -1.367 0.171811    
## stat159      3.893e-05  2.466e-04   0.158 0.874567    
## stat160      1.321e-04  2.501e-04   0.528 0.597470    
## stat161      3.025e-04  2.499e-04   1.211 0.226136    
## stat162      2.252e-07  2.464e-04   0.001 0.999271    
## stat163      8.888e-05  2.511e-04   0.354 0.723377    
## stat164      2.484e-05  2.496e-04   0.100 0.920715    
## stat165      4.252e-05  2.472e-04   0.172 0.863436    
## stat166     -1.694e-04  2.456e-04  -0.690 0.490394    
## stat167     -1.931e-04  2.485e-04  -0.777 0.437201    
## stat168     -1.567e-04  2.478e-04  -0.632 0.527182    
## stat169      4.297e-05  2.476e-04   0.174 0.862214    
## stat170     -3.394e-04  2.482e-04  -1.368 0.171506    
## stat171      3.097e-05  2.507e-04   0.124 0.901684    
## stat172      2.167e-04  2.478e-04   0.875 0.381723    
## stat173     -3.855e-04  2.500e-04  -1.542 0.123184    
## stat174      1.224e-04  2.494e-04   0.491 0.623580    
## stat175     -5.229e-04  2.478e-04  -2.110 0.034862 *  
## stat176      1.117e-04  2.473e-04   0.452 0.651545    
## stat177      5.684e-05  2.489e-04   0.228 0.819410    
## stat178     -8.145e-05  2.521e-04  -0.323 0.746664    
## stat179     -3.494e-05  2.494e-04  -0.140 0.888607    
## stat180     -2.653e-04  2.471e-04  -1.074 0.283033    
## stat181      3.433e-04  2.497e-04   1.375 0.169302    
## stat182      3.930e-05  2.502e-04   0.157 0.875155    
## stat183      2.992e-04  2.476e-04   1.208 0.226917    
## stat184     -2.532e-05  2.496e-04  -0.101 0.919207    
## stat185     -1.709e-04  2.461e-04  -0.695 0.487325    
## stat186     -2.002e-04  2.497e-04  -0.802 0.422708    
## stat187     -3.295e-04  2.476e-04  -1.331 0.183332    
## stat188     -2.290e-04  2.476e-04  -0.925 0.355130    
## stat189      3.384e-04  2.482e-04   1.364 0.172670    
## stat190      6.180e-05  2.467e-04   0.251 0.802170    
## stat191     -1.058e-04  2.486e-04  -0.426 0.670480    
## stat192     -2.813e-04  2.514e-04  -1.119 0.263169    
## stat193     -4.199e-05  2.525e-04  -0.166 0.867963    
## stat194     -9.017e-05  2.479e-04  -0.364 0.716076    
## stat195      4.201e-04  2.480e-04   1.694 0.090350 .  
## stat196     -2.143e-04  2.518e-04  -0.851 0.394716    
## stat197      1.269e-04  2.461e-04   0.516 0.606071    
## stat198     -2.250e-04  2.491e-04  -0.903 0.366405    
## stat199      1.989e-04  2.464e-04   0.807 0.419435    
## stat200     -3.141e-04  2.465e-04  -1.274 0.202750    
## stat201      2.323e-05  2.485e-04   0.093 0.925535    
## stat202      7.254e-05  2.515e-04   0.288 0.773066    
## stat203      1.590e-04  2.480e-04   0.641 0.521505    
## stat204     -5.073e-04  2.462e-04  -2.060 0.039421 *  
## stat205     -2.776e-05  2.479e-04  -0.112 0.910850    
## stat206     -1.054e-04  2.489e-04  -0.424 0.671789    
## stat207      5.098e-04  2.482e-04   2.054 0.040009 *  
## stat208     -2.161e-04  2.487e-04  -0.869 0.384999    
## stat209      4.761e-05  2.464e-04   0.193 0.846812    
## stat210     -1.111e-04  2.491e-04  -0.446 0.655635    
## stat211     -2.100e-04  2.485e-04  -0.845 0.398082    
## stat212      7.678e-05  2.479e-04   0.310 0.756820    
## stat213     -1.064e-04  2.507e-04  -0.424 0.671322    
## stat214     -6.761e-05  2.494e-04  -0.271 0.786315    
## stat215     -1.704e-04  2.489e-04  -0.684 0.493712    
## stat216     -1.489e-04  2.495e-04  -0.597 0.550610    
## stat217      1.815e-04  2.489e-04   0.729 0.465742    
## x18.sqrt     2.657e-02  9.510e-04  27.943  < 2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 0.03144 on 5343 degrees of freedom
## Multiple R-squared:  0.2711, Adjusted R-squared:  0.2384 
## F-statistic:  8.28 on 240 and 5343 DF,  p-value: < 2.2e-16
cd.full = plot.diagnostics(model=model.full, train=data.train)

## [1] "Number of data points that have Cook's D > 4/n: 287"
## [1] "Number of data points that have Cook's D > 1: 0"

Checking with removal of high influence points

high.cd = names(cd.full[cd.full > 4/nrow(data.train)])
data.train2 = data.train[!(rownames(data.train)) %in% high.cd,]
model.full2 = lm(formula , data.train2)
summary(model.full2)
## 
## Call:
## lm(formula = formula, data = data.train2)
## 
## Residuals:
##       Min        1Q    Median        3Q       Max 
## -0.059294 -0.017046 -0.002591  0.015983  0.067490 
## 
## Coefficients:
##               Estimate Std. Error t value Pr(>|t|)    
## (Intercept)  1.956e+00  7.812e-03 250.385  < 2e-16 ***
## x1          -1.494e-04  5.363e-04  -0.279 0.780566    
## x2           4.069e-04  3.391e-04   1.200 0.230321    
## x3           2.805e-05  9.315e-05   0.301 0.763333    
## x4          -5.760e-05  7.380e-06  -7.804 7.22e-15 ***
## x5           3.485e-04  2.403e-04   1.451 0.146948    
## x6          -3.370e-04  4.860e-04  -0.693 0.488103    
## x7           1.254e-02  5.223e-04  24.008  < 2e-16 ***
## x8           4.653e-04  1.215e-04   3.830 0.000130 ***
## x9           3.143e-03  2.693e-04  11.673  < 2e-16 ***
## x10          1.520e-03  2.515e-04   6.042 1.63e-09 ***
## x11          2.343e+05  6.019e+04   3.893 0.000100 ***
## x12          1.017e-04  1.533e-04   0.664 0.506882    
## x13          9.145e-05  6.111e-05   1.497 0.134574    
## x14         -1.790e-04  2.622e-04  -0.683 0.494764    
## x15          1.980e-04  2.517e-04   0.786 0.431689    
## x16          8.441e-04  1.753e-04   4.814 1.52e-06 ***
## x17          1.608e-03  2.653e-04   6.062 1.44e-09 ***
## x19          1.183e-04  1.355e-04   0.873 0.382848    
## x20         -1.064e-03  9.345e-04  -1.139 0.254830    
## x21          1.289e-04  3.438e-05   3.751 0.000178 ***
## x22         -6.740e-04  2.811e-04  -2.398 0.016522 *  
## x23          1.429e-04  2.679e-04   0.533 0.593889    
## stat1       -3.759e-04  2.028e-04  -1.854 0.063844 .  
## stat2        3.122e-04  2.021e-04   1.545 0.122493    
## stat3        5.959e-04  2.028e-04   2.939 0.003308 ** 
## stat4       -6.972e-04  2.032e-04  -3.431 0.000606 ***
## stat5       -3.489e-04  2.045e-04  -1.706 0.088019 .  
## stat6       -3.497e-04  2.045e-04  -1.711 0.087231 .  
## stat7       -2.206e-04  2.034e-04  -1.085 0.278044    
## stat8        6.409e-05  2.017e-04   0.318 0.750630    
## stat9        5.991e-06  2.020e-04   0.030 0.976347    
## stat10      -2.520e-04  2.022e-04  -1.247 0.212606    
## stat11      -3.649e-04  2.035e-04  -1.793 0.073040 .  
## stat12       1.648e-04  2.027e-04   0.813 0.416221    
## stat13      -5.516e-04  2.021e-04  -2.730 0.006358 ** 
## stat14      -8.512e-04  2.016e-04  -4.223 2.45e-05 ***
## stat15      -4.527e-04  2.006e-04  -2.257 0.024067 *  
## stat16      -1.561e-04  2.029e-04  -0.769 0.441735    
## stat17      -2.371e-05  2.011e-04  -0.118 0.906120    
## stat18      -2.756e-04  2.014e-04  -1.368 0.171313    
## stat19       1.392e-04  2.018e-04   0.690 0.490531    
## stat20       1.636e-04  2.019e-04   0.810 0.417868    
## stat21      -3.948e-05  2.027e-04  -0.195 0.845602    
## stat22      -3.209e-04  2.035e-04  -1.577 0.114807    
## stat23       5.335e-04  2.034e-04   2.623 0.008739 ** 
## stat24      -4.139e-04  2.025e-04  -2.044 0.040998 *  
## stat25      -2.471e-04  2.023e-04  -1.221 0.222032    
## stat26      -4.291e-04  2.020e-04  -2.125 0.033676 *  
## stat27       4.196e-06  2.034e-04   0.021 0.983547    
## stat28      -1.837e-04  2.028e-04  -0.906 0.364977    
## stat29      -5.357e-05  2.027e-04  -0.264 0.791536    
## stat30       2.976e-04  2.032e-04   1.465 0.143003    
## stat31       8.422e-05  2.041e-04   0.413 0.679953    
## stat32      -3.583e-05  2.046e-04  -0.175 0.860992    
## stat33      -1.443e-05  2.013e-04  -0.072 0.942857    
## stat34       2.803e-04  2.011e-04   1.394 0.163425    
## stat35      -6.038e-04  2.037e-04  -2.964 0.003047 ** 
## stat36       8.904e-06  2.019e-04   0.044 0.964820    
## stat37      -1.547e-04  2.036e-04  -0.760 0.447347    
## stat38       3.535e-04  2.023e-04   1.748 0.080591 .  
## stat39      -3.814e-04  2.010e-04  -1.898 0.057800 .  
## stat40      -8.625e-05  2.033e-04  -0.424 0.671374    
## stat41      -4.174e-04  2.017e-04  -2.069 0.038552 *  
## stat42      -1.087e-04  2.025e-04  -0.537 0.591577    
## stat43      -2.263e-04  2.036e-04  -1.111 0.266512    
## stat44      -1.266e-04  2.018e-04  -0.627 0.530473    
## stat45      -2.903e-04  2.019e-04  -1.438 0.150576    
## stat46       2.531e-04  2.028e-04   1.248 0.212086    
## stat47       3.297e-04  2.047e-04   1.610 0.107380    
## stat48       3.389e-04  2.019e-04   1.679 0.093264 .  
## stat49      -1.290e-04  2.005e-04  -0.644 0.519905    
## stat50       2.207e-04  2.008e-04   1.099 0.271948    
## stat51       2.663e-04  2.015e-04   1.322 0.186277    
## stat52      -2.137e-04  2.025e-04  -1.055 0.291389    
## stat53      -3.916e-04  2.041e-04  -1.918 0.055159 .  
## stat54      -3.708e-04  2.052e-04  -1.807 0.070830 .  
## stat55      -4.204e-05  2.012e-04  -0.209 0.834501    
## stat56      -9.247e-05  2.036e-04  -0.454 0.649765    
## stat57       1.157e-05  2.008e-04   0.058 0.954054    
## stat58       1.207e-04  2.017e-04   0.599 0.549434    
## stat59       4.467e-04  2.023e-04   2.208 0.027286 *  
## stat60       5.146e-04  2.031e-04   2.534 0.011322 *  
## stat61      -1.654e-04  2.036e-04  -0.812 0.416749    
## stat62      -3.195e-04  2.013e-04  -1.588 0.112454    
## stat63       1.401e-04  2.033e-04   0.689 0.490832    
## stat64       3.163e-05  2.007e-04   0.158 0.874766    
## stat65      -1.571e-04  2.023e-04  -0.777 0.437429    
## stat66       1.525e-04  2.042e-04   0.747 0.455392    
## stat67       8.752e-05  2.029e-04   0.431 0.666274    
## stat68      -1.232e-04  2.031e-04  -0.607 0.544097    
## stat69      -1.111e-05  2.036e-04  -0.055 0.956499    
## stat70       2.771e-04  2.015e-04   1.375 0.169234    
## stat71       3.878e-04  2.015e-04   1.925 0.054269 .  
## stat72       5.745e-05  2.035e-04   0.282 0.777706    
## stat73      -5.904e-06  2.037e-04  -0.029 0.976878    
## stat74      -1.465e-04  2.026e-04  -0.723 0.469612    
## stat75      -1.788e-05  2.036e-04  -0.088 0.930025    
## stat76       2.976e-04  2.033e-04   1.464 0.143282    
## stat77       7.013e-05  2.020e-04   0.347 0.728457    
## stat78      -2.366e-04  2.015e-04  -1.174 0.240501    
## stat79       5.550e-05  2.035e-04   0.273 0.785065    
## stat80       3.362e-04  2.034e-04   1.653 0.098434 .  
## stat81      -3.485e-05  2.032e-04  -0.171 0.863845    
## stat82       4.269e-05  2.017e-04   0.212 0.832359    
## stat83      -5.930e-05  2.026e-04  -0.293 0.769755    
## stat84      -4.600e-04  2.017e-04  -2.281 0.022599 *  
## stat85      -3.382e-04  2.028e-04  -1.667 0.095500 .  
## stat86       2.601e-04  2.032e-04   1.280 0.200545    
## stat87      -3.378e-04  2.031e-04  -1.663 0.096418 .  
## stat88      -1.510e-04  2.005e-04  -0.753 0.451219    
## stat89       1.163e-04  2.012e-04   0.578 0.563275    
## stat90      -5.436e-05  2.043e-04  -0.266 0.790175    
## stat91      -4.891e-04  2.001e-04  -2.445 0.014515 *  
## stat92      -2.037e-04  2.017e-04  -1.010 0.312678    
## stat93      -2.020e-04  2.046e-04  -0.987 0.323573    
## stat94      -1.499e-04  2.038e-04  -0.735 0.462165    
## stat95       1.317e-04  2.034e-04   0.647 0.517463    
## stat96      -2.207e-04  2.018e-04  -1.094 0.274170    
## stat97       2.967e-04  2.003e-04   1.481 0.138575    
## stat98       3.514e-03  2.001e-04  17.559  < 2e-16 ***
## stat99       3.738e-04  2.036e-04   1.836 0.066377 .  
## stat100      5.945e-04  2.037e-04   2.919 0.003532 ** 
## stat101     -3.370e-05  2.045e-04  -0.165 0.869116    
## stat102      4.645e-05  2.039e-04   0.228 0.819768    
## stat103     -5.200e-04  2.038e-04  -2.552 0.010748 *  
## stat104     -1.820e-04  2.022e-04  -0.900 0.367935    
## stat105      2.493e-04  2.016e-04   1.237 0.216313    
## stat106     -4.238e-04  2.014e-04  -2.105 0.035375 *  
## stat107      1.414e-04  2.013e-04   0.703 0.482262    
## stat108     -1.754e-04  2.021e-04  -0.868 0.385652    
## stat109     -2.405e-04  2.025e-04  -1.188 0.234882    
## stat110     -3.305e-03  2.015e-04 -16.400  < 2e-16 ***
## stat111     -1.203e-04  2.007e-04  -0.599 0.549008    
## stat112     -7.819e-05  2.041e-04  -0.383 0.701637    
## stat113     -3.453e-04  2.045e-04  -1.688 0.091472 .  
## stat114      2.238e-04  2.028e-04   1.104 0.269661    
## stat115      3.974e-04  2.024e-04   1.964 0.049589 *  
## stat116      3.616e-04  2.036e-04   1.776 0.075818 .  
## stat117      2.611e-04  2.033e-04   1.285 0.198980    
## stat118      2.419e-04  2.005e-04   1.207 0.227667    
## stat119      7.340e-05  2.011e-04   0.365 0.715140    
## stat120     -1.011e-05  2.008e-04  -0.050 0.959851    
## stat121     -3.101e-04  2.023e-04  -1.533 0.125381    
## stat122     -8.462e-05  2.020e-04  -0.419 0.675235    
## stat123      2.160e-04  2.046e-04   1.055 0.291247    
## stat124     -2.198e-04  2.014e-04  -1.092 0.275000    
## stat125      2.309e-04  2.025e-04   1.141 0.254109    
## stat126      7.555e-05  2.015e-04   0.375 0.707785    
## stat127     -1.232e-04  2.023e-04  -0.609 0.542574    
## stat128     -3.543e-04  2.023e-04  -1.752 0.079871 .  
## stat129      8.300e-05  2.022e-04   0.410 0.681534    
## stat130      1.375e-04  2.018e-04   0.682 0.495565    
## stat131      2.901e-04  2.027e-04   1.431 0.152506    
## stat132      4.974e-05  2.016e-04   0.247 0.805136    
## stat133      4.503e-05  2.033e-04   0.221 0.824743    
## stat134     -2.712e-04  2.005e-04  -1.352 0.176296    
## stat135     -1.521e-04  2.017e-04  -0.754 0.450833    
## stat136     -4.646e-05  2.034e-04  -0.228 0.819344    
## stat137      2.611e-04  2.016e-04   1.295 0.195343    
## stat138      2.511e-05  2.019e-04   0.124 0.901030    
## stat139     -3.942e-05  2.033e-04  -0.194 0.846234    
## stat140      2.411e-04  2.002e-04   1.205 0.228428    
## stat141      1.732e-04  2.006e-04   0.863 0.388138    
## stat142      1.680e-04  2.034e-04   0.826 0.408810    
## stat143      5.512e-05  2.021e-04   0.273 0.785043    
## stat144      3.074e-04  2.017e-04   1.524 0.127487    
## stat145     -2.543e-05  2.055e-04  -0.124 0.901493    
## stat146     -5.168e-04  2.038e-04  -2.536 0.011235 *  
## stat147     -2.539e-04  2.038e-04  -1.246 0.212889    
## stat148     -2.830e-04  2.008e-04  -1.409 0.158837    
## stat149     -5.721e-04  2.037e-04  -2.809 0.004995 ** 
## stat150     -6.187e-05  2.027e-04  -0.305 0.760232    
## stat151      4.243e-04  2.059e-04   2.061 0.039345 *  
## stat152     -1.063e-04  2.005e-04  -0.530 0.596121    
## stat153      1.582e-04  2.053e-04   0.770 0.441120    
## stat154      3.014e-04  2.048e-04   1.472 0.141131    
## stat155     -1.970e-05  2.019e-04  -0.098 0.922276    
## stat156      3.626e-04  2.037e-04   1.780 0.075171 .  
## stat157     -9.273e-05  2.006e-04  -0.462 0.643976    
## stat158      6.342e-05  2.043e-04   0.310 0.756311    
## stat159     -2.653e-05  2.007e-04  -0.132 0.894813    
## stat160      1.751e-05  2.046e-04   0.086 0.931810    
## stat161      2.126e-04  2.036e-04   1.045 0.296294    
## stat162     -4.630e-05  2.005e-04  -0.231 0.817345    
## stat163      6.435e-05  2.056e-04   0.313 0.754305    
## stat164     -6.937e-05  2.038e-04  -0.340 0.733535    
## stat165      1.262e-04  2.016e-04   0.626 0.531305    
## stat166     -1.139e-04  1.998e-04  -0.570 0.568651    
## stat167     -1.887e-04  2.024e-04  -0.932 0.351243    
## stat168     -8.164e-05  2.017e-04  -0.405 0.685705    
## stat169      5.577e-05  2.021e-04   0.276 0.782607    
## stat170     -2.233e-04  2.026e-04  -1.102 0.270392    
## stat171     -1.051e-04  2.042e-04  -0.515 0.606739    
## stat172      4.069e-04  2.015e-04   2.019 0.043516 *  
## stat173     -2.295e-04  2.033e-04  -1.129 0.258859    
## stat174      3.115e-04  2.029e-04   1.535 0.124772    
## stat175     -5.036e-04  2.015e-04  -2.499 0.012497 *  
## stat176     -1.566e-04  2.012e-04  -0.778 0.436329    
## stat177     -2.819e-04  2.029e-04  -1.389 0.164784    
## stat178      1.962e-06  2.056e-04   0.010 0.992383    
## stat179     -9.906e-05  2.033e-04  -0.487 0.626097    
## stat180     -1.372e-04  2.021e-04  -0.679 0.497372    
## stat181      4.665e-04  2.034e-04   2.293 0.021884 *  
## stat182      1.694e-04  2.044e-04   0.829 0.407420    
## stat183      3.423e-04  2.021e-04   1.694 0.090390 .  
## stat184      2.245e-04  2.033e-04   1.104 0.269631    
## stat185      3.932e-05  2.005e-04   0.196 0.844554    
## stat186      5.058e-05  2.034e-04   0.249 0.803680    
## stat187     -1.384e-04  2.016e-04  -0.686 0.492470    
## stat188     -2.200e-05  2.021e-04  -0.109 0.913336    
## stat189      1.758e-04  2.024e-04   0.869 0.385015    
## stat190     -1.373e-04  2.010e-04  -0.683 0.494588    
## stat191     -9.242e-05  2.027e-04  -0.456 0.648501    
## stat192     -1.098e-04  2.052e-04  -0.535 0.592799    
## stat193      6.404e-05  2.061e-04   0.311 0.756070    
## stat194     -1.452e-04  2.021e-04  -0.718 0.472520    
## stat195      1.091e-04  2.023e-04   0.540 0.589488    
## stat196     -3.111e-04  2.048e-04  -1.519 0.128849    
## stat197     -6.126e-05  2.008e-04  -0.305 0.760359    
## stat198     -1.175e-04  2.025e-04  -0.580 0.561638    
## stat199      9.900e-05  2.006e-04   0.493 0.621708    
## stat200     -2.997e-04  2.012e-04  -1.489 0.136426    
## stat201      1.649e-05  2.032e-04   0.081 0.935305    
## stat202      1.280e-04  2.049e-04   0.625 0.532279    
## stat203      1.340e-04  2.025e-04   0.662 0.508051    
## stat204     -2.342e-04  2.007e-04  -1.167 0.243222    
## stat205      1.607e-04  2.014e-04   0.798 0.424800    
## stat206     -2.524e-04  2.028e-04  -1.244 0.213379    
## stat207      4.682e-04  2.023e-04   2.314 0.020703 *  
## stat208     -1.262e-04  2.031e-04  -0.621 0.534436    
## stat209      9.184e-05  2.004e-04   0.458 0.646838    
## stat210     -2.386e-04  2.027e-04  -1.177 0.239244    
## stat211     -1.381e-04  2.027e-04  -0.681 0.495646    
## stat212      1.012e-04  2.021e-04   0.500 0.616787    
## stat213     -8.137e-05  2.042e-04  -0.398 0.690318    
## stat214      1.315e-04  2.037e-04   0.646 0.518508    
## stat215     -1.134e-04  2.033e-04  -0.558 0.576925    
## stat216     -2.369e-04  2.030e-04  -1.167 0.243319    
## stat217      8.349e-05  2.025e-04   0.412 0.680122    
## x18.sqrt     2.689e-02  7.730e-04  34.783  < 2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 0.02493 on 5056 degrees of freedom
## Multiple R-squared:  0.3864, Adjusted R-squared:  0.3573 
## F-statistic: 13.27 on 240 and 5056 DF,  p-value: < 2.2e-16
cd.full2 = plot.diagnostics(model.full2, data.train2)

## [1] "Number of data points that have Cook's D > 4/n: 269"
## [1] "Number of data points that have Cook's D > 1: 0"
# much more normal residuals than before. 
# Checking to see if distributions are different and if so whcih variables
# High Leverage Plot 
plotData = data.train %>% 
  rownames_to_column() %>%
  mutate(type=ifelse(rowname %in% high.cd,'High','Normal')) %>%
  dplyr::select(type,target=one_of(label.names))

ggplot(data=plotData, aes(x=type,y=target)) +
  geom_boxplot(fill='light blue',outlier.shape=NA) +
  scale_y_continuous(name="Target Variable Values",label=scales::comma_format(accuracy=.1)) +
  theme_light() +
  ggtitle('Distribution of High Leverage Points and Normal  Points')

# 2 sample t-tests

plotData = data.train %>% 
  rownames_to_column() %>%
  mutate(type=ifelse(rowname %in% high.cd,'High','Normal')) %>%
  dplyr::select(type,one_of(feature.names))

comp.test = lapply(dplyr::select(plotData, one_of(feature.names))
                   , function(x) t.test(x ~ plotData$type, var.equal = TRUE)) 

sig.comp = list.filter(comp.test, p.value < 0.05)
sapply(sig.comp, function(x) x[['p.value']])
##           x4       stat15       stat19       stat38       stat49       stat82       stat98      stat101      stat110 
## 2.772613e-02 4.232345e-02 3.891725e-02 2.182942e-02 2.114386e-02 2.135716e-02 3.618666e-06 4.293858e-02 1.151664e-04 
##      stat128      stat144      stat146      stat151      stat158     x18.sqrt 
## 2.243626e-02 4.705614e-02 3.530774e-02 4.747317e-02 7.249900e-03 5.033029e-03
mm = melt(plotData, id=c('type')) %>% filter(variable %in% names(sig.comp))

ggplot(mm,aes(x=type, y=value)) +
  geom_boxplot()+
  facet_wrap(~variable, ncol=5, scales = 'free_y') +
  scale_y_continuous(name="values",label=scales::comma_format(accuracy=.1)) +
  ggtitle('Distribution of High Leverage Points and Normal Points')

# Distribution (box) Plots
mm = melt(plotData, id=c('type'))

ggplot(mm,aes(x=type, y=value)) +
  geom_boxplot()+
  facet_wrap(~variable, ncol=8, scales = 'free_y') +
  scale_y_continuous(name="values",label=scales::comma_format(accuracy=.1)) +
  ggtitle('Distribution of High Leverage Points and Normal Points')

Grand Means Model

model.null = lm(grand.mean.formula, data.train)
summary(model.null)
## 
## Call:
## lm(formula = grand.mean.formula, data = data.train)
## 
## Residuals:
##       Min        1Q    Median        3Q       Max 
## -0.114767 -0.023878 -0.002993  0.020728  0.190546 
## 
## Coefficients:
##              Estimate Std. Error t value Pr(>|t|)    
## (Intercept) 2.0966427  0.0004822    4349   <2e-16 ***
## ---
## Signif. codes:  0 '***' 0.001 '**' 0.01 '*' 0.05 '.' 0.1 ' ' 1
## 
## Residual standard error: 0.03603 on 5583 degrees of freedom

Variable Selection

Basic: http://www.stat.columbia.edu/~martin/W2024/R10.pdf Cross Validation + Other Metrics: http://www.sthda.com/english/articles/37-model-selection-essentials-in-r/154-stepwise-regression-essentials-in-r/

Forward Selection with CV

Train

if (algo.forward.caret == TRUE){
  set.seed(1)
  returned = train.caret.glmselect(formula = formula
                                   , data = data.train
                                   , method = "leapForward"
                                   , feature.names = feature.names)
  model.forward = returned$model
  id = returned$id
}
## Aggregating results
## Selecting tuning parameters
## Fitting nvmax = 10 on full training set
## [1] "All models results"
##     nvmax       RMSE  Rsquared        MAE       RMSESD RsquaredSD        MAESD
## 1       1 0.03394945 0.1136866 0.02639879 0.0009254924 0.02842538 0.0007668152
## 2       2 0.03313811 0.1544642 0.02563570 0.0008856904 0.02102133 0.0006503708
## 3       3 0.03252158 0.1856417 0.02506403 0.0009775242 0.02462482 0.0007440002
## 4       4 0.03203850 0.2095422 0.02440380 0.0010257703 0.02350731 0.0006318352
## 5       5 0.03175204 0.2236513 0.02420630 0.0010473102 0.02660189 0.0006818957
## 6       6 0.03165743 0.2282201 0.02412261 0.0009917289 0.02542266 0.0006188139
## 7       7 0.03167257 0.2274306 0.02415804 0.0009624469 0.02634536 0.0006081113
## 8       8 0.03160157 0.2308579 0.02409588 0.0009524258 0.02838942 0.0005958236
## 9       9 0.03157712 0.2319660 0.02408496 0.0009391885 0.02778037 0.0005830139
## 10     10 0.03156674 0.2325072 0.02407735 0.0009401267 0.02752636 0.0005792342
## 11     11 0.03157638 0.2320532 0.02409075 0.0009407004 0.02688228 0.0005850813
## 12     12 0.03159046 0.2314010 0.02412861 0.0009277078 0.02697054 0.0005820300
## 13     13 0.03157870 0.2319915 0.02411271 0.0009526699 0.02639659 0.0006047196
## 14     14 0.03158038 0.2319175 0.02409986 0.0009579561 0.02687828 0.0006030872
## 15     15 0.03161145 0.2305226 0.02412651 0.0009462155 0.02846025 0.0005743454
## 16     16 0.03160880 0.2306841 0.02412201 0.0009355650 0.02774726 0.0005601936
## 17     17 0.03161587 0.2303909 0.02412916 0.0009325726 0.02777345 0.0005573390
## 18     18 0.03163837 0.2294505 0.02415444 0.0009470147 0.02882990 0.0005608970
## 19     19 0.03164803 0.2290008 0.02414618 0.0009519633 0.02886538 0.0005868361
## 20     20 0.03166056 0.2284943 0.02415060 0.0009848732 0.02954986 0.0006246196
## 21     21 0.03167944 0.2276271 0.02415464 0.0009861291 0.02926543 0.0006240104
## 22     22 0.03171628 0.2259653 0.02418771 0.0009997310 0.02930460 0.0006348048
## 23     23 0.03171499 0.2260902 0.02417814 0.0009963089 0.02886254 0.0006574192
## 24     24 0.03173080 0.2253290 0.02417598 0.0009888086 0.02780067 0.0006512763
## 25     25 0.03172739 0.2255470 0.02417852 0.0009895676 0.02801837 0.0006512339
## 26     26 0.03173581 0.2252134 0.02419314 0.0009875325 0.02840859 0.0006472089
## 27     27 0.03176140 0.2240225 0.02421124 0.0010006864 0.02918309 0.0006731392
## 28     28 0.03178149 0.2231374 0.02423047 0.0010142650 0.02932885 0.0006740206
## 29     29 0.03177985 0.2232506 0.02423167 0.0009979302 0.02857497 0.0006782857
## 30     30 0.03179749 0.2224423 0.02424979 0.0009833970 0.02837964 0.0006707433
## 31     31 0.03179845 0.2223712 0.02424822 0.0009764295 0.02727671 0.0006702864
## 32     32 0.03179905 0.2223781 0.02423892 0.0009860655 0.02752385 0.0006755555
## 33     33 0.03181046 0.2218473 0.02425145 0.0009957706 0.02695787 0.0006885075
## 34     34 0.03181808 0.2215404 0.02426286 0.0009961237 0.02670827 0.0006869929
## 35     35 0.03183409 0.2208456 0.02427971 0.0009866623 0.02604409 0.0006746184
## 36     36 0.03184566 0.2203686 0.02428748 0.0009978540 0.02638850 0.0006644015
## 37     37 0.03186002 0.2197726 0.02430243 0.0009979233 0.02708057 0.0006618313
## 38     38 0.03185286 0.2201032 0.02429743 0.0010070158 0.02703542 0.0006770484
## 39     39 0.03185931 0.2198429 0.02429195 0.0009979542 0.02711419 0.0006770475
## 40     40 0.03187286 0.2192320 0.02430268 0.0010068214 0.02701335 0.0006808964
## 41     41 0.03187959 0.2189213 0.02430666 0.0010079817 0.02648068 0.0006755842
## 42     42 0.03188482 0.2186789 0.02430243 0.0009866603 0.02610080 0.0006583586
## 43     43 0.03189040 0.2184824 0.02430493 0.0009675667 0.02612758 0.0006331428
## 44     44 0.03189218 0.2184396 0.02430455 0.0009779052 0.02616928 0.0006514381
## 45     45 0.03190036 0.2180863 0.02430574 0.0009730793 0.02621815 0.0006399133
## 46     46 0.03191723 0.2173437 0.02432239 0.0009742216 0.02613060 0.0006403279
## 47     47 0.03192640 0.2169635 0.02433304 0.0009687790 0.02644793 0.0006325208
## 48     48 0.03193261 0.2166994 0.02433596 0.0009705550 0.02632107 0.0006376387
## 49     49 0.03192711 0.2169825 0.02432389 0.0009788934 0.02653843 0.0006489526
## 50     50 0.03193657 0.2166221 0.02433055 0.0009749169 0.02701476 0.0006565543
## 51     51 0.03194300 0.2163467 0.02434149 0.0009716772 0.02656678 0.0006463491
## 52     52 0.03193572 0.2167099 0.02433990 0.0009755533 0.02709201 0.0006536793
## 53     53 0.03194672 0.2162787 0.02435158 0.0009668386 0.02682795 0.0006524253
## 54     54 0.03195326 0.2160017 0.02435372 0.0009789109 0.02675347 0.0006461757
## 55     55 0.03196320 0.2156391 0.02436509 0.0009757962 0.02694840 0.0006316139
## 56     56 0.03197671 0.2150945 0.02436768 0.0009737023 0.02708730 0.0006165927
## 57     57 0.03197372 0.2152633 0.02436357 0.0009687470 0.02713808 0.0006079834
## 58     58 0.03199071 0.2145281 0.02436853 0.0009654312 0.02724889 0.0006052307
## 59     59 0.03198879 0.2145867 0.02437393 0.0009844774 0.02689940 0.0006119151
## 60     60 0.03199795 0.2142147 0.02438918 0.0009792211 0.02683672 0.0006101987
## 61     61 0.03200550 0.2138830 0.02439269 0.0009738843 0.02652444 0.0006070169
## 62     62 0.03201215 0.2135535 0.02440707 0.0009680793 0.02615914 0.0006092096
## 63     63 0.03202028 0.2132123 0.02440998 0.0009714807 0.02664895 0.0006094080
## 64     64 0.03202521 0.2130129 0.02441759 0.0009750510 0.02681277 0.0006049835
## 65     65 0.03202236 0.2131600 0.02442293 0.0009843453 0.02689799 0.0006046425
## 66     66 0.03202909 0.2128975 0.02442153 0.0009769514 0.02676496 0.0006124038
## 67     67 0.03203617 0.2125732 0.02442304 0.0009655816 0.02670065 0.0006087370
## 68     68 0.03203852 0.2125096 0.02442681 0.0009619549 0.02668387 0.0006052416
## 69     69 0.03203853 0.2125548 0.02443260 0.0009641843 0.02638979 0.0006108769
## 70     70 0.03203970 0.2125244 0.02443044 0.0009819977 0.02678385 0.0006347403
## 71     71 0.03203550 0.2127192 0.02442528 0.0009913815 0.02716994 0.0006375596
## 72     72 0.03204357 0.2124469 0.02442944 0.0009915970 0.02714195 0.0006412472
## 73     73 0.03203968 0.2126520 0.02442316 0.0009930209 0.02705819 0.0006438160
## 74     74 0.03204292 0.2125206 0.02442373 0.0010082107 0.02806019 0.0006536032
## 75     75 0.03204991 0.2122962 0.02442715 0.0010107003 0.02822597 0.0006551940
## 76     76 0.03204247 0.2126474 0.02441811 0.0010206838 0.02788966 0.0006717178
## 77     77 0.03203761 0.2128916 0.02441020 0.0010268890 0.02801852 0.0006694037
## 78     78 0.03204653 0.2125362 0.02441771 0.0010367210 0.02793240 0.0006749455
## 79     79 0.03205867 0.2120514 0.02442260 0.0010235579 0.02767060 0.0006731984
## 80     80 0.03205721 0.2121859 0.02441808 0.0010289038 0.02780988 0.0006774063
## 81     81 0.03206929 0.2116541 0.02443052 0.0010164678 0.02782611 0.0006650937
## 82     82 0.03206942 0.2116629 0.02443117 0.0010193246 0.02787106 0.0006546312
## 83     83 0.03207160 0.2116122 0.02442767 0.0010186039 0.02793876 0.0006462783
## 84     84 0.03207477 0.2115027 0.02442802 0.0010251764 0.02814857 0.0006498159
## 85     85 0.03208658 0.2109946 0.02443620 0.0010254181 0.02769567 0.0006590273
## 86     86 0.03208724 0.2109282 0.02443719 0.0010217087 0.02756520 0.0006566185
## 87     87 0.03209168 0.2107912 0.02444549 0.0010210306 0.02766041 0.0006508071
## 88     88 0.03209496 0.2106513 0.02444919 0.0010214717 0.02754500 0.0006538201
## 89     89 0.03209548 0.2105810 0.02445317 0.0010195560 0.02756915 0.0006568803
## 90     90 0.03210172 0.2103121 0.02445389 0.0010154316 0.02738604 0.0006530532
## 91     91 0.03211034 0.2100224 0.02445938 0.0010070948 0.02734864 0.0006517491
## 92     92 0.03211011 0.2100833 0.02445747 0.0010045174 0.02745285 0.0006425260
## 93     93 0.03210326 0.2103744 0.02445663 0.0010104026 0.02747248 0.0006532665
## 94     94 0.03211088 0.2100454 0.02446002 0.0010095792 0.02741675 0.0006523079
## 95     95 0.03211627 0.2098045 0.02446547 0.0009981894 0.02706359 0.0006356527
## 96     96 0.03213436 0.2090406 0.02448205 0.0010016062 0.02740688 0.0006310896
## 97     97 0.03213101 0.2091754 0.02448456 0.0010016673 0.02715521 0.0006266564
## 98     98 0.03213583 0.2089647 0.02449029 0.0009979592 0.02676393 0.0006278837
## 99     99 0.03215043 0.2083663 0.02449864 0.0009984826 0.02682318 0.0006268662
## 100   100 0.03215106 0.2083584 0.02449983 0.0009973685 0.02657902 0.0006297643
## 101   101 0.03215915 0.2080238 0.02450403 0.0010010652 0.02675964 0.0006287615
## 102   102 0.03215959 0.2080206 0.02450726 0.0010049889 0.02706353 0.0006264374
## 103   103 0.03215509 0.2082783 0.02449936 0.0010038500 0.02749196 0.0006266728
## 104   104 0.03215230 0.2084127 0.02450064 0.0009993471 0.02768638 0.0006251569
## 105   105 0.03214973 0.2085434 0.02449696 0.0009975507 0.02743019 0.0006159073
## 106   106 0.03215373 0.2083743 0.02449924 0.0009965149 0.02732921 0.0006203764
## 107   107 0.03215433 0.2083658 0.02449652 0.0009980852 0.02756543 0.0006138287
## 108   108 0.03215566 0.2083235 0.02449694 0.0009952907 0.02774223 0.0006072742
## 109   109 0.03215032 0.2085880 0.02449966 0.0009961211 0.02780315 0.0006082481
## 110   110 0.03214901 0.2086334 0.02450052 0.0010015034 0.02802072 0.0006103756
## 111   111 0.03215319 0.2084662 0.02450355 0.0010004725 0.02782824 0.0006106944
## 112   112 0.03215787 0.2082905 0.02450306 0.0010063720 0.02809041 0.0006148493
## 113   113 0.03214973 0.2086418 0.02449862 0.0010051546 0.02774904 0.0006164905
## 114   114 0.03214178 0.2089776 0.02449803 0.0010086693 0.02778729 0.0006174565
## 115   115 0.03214014 0.2090633 0.02449237 0.0010077077 0.02740816 0.0006183037
## 116   116 0.03213835 0.2091388 0.02449902 0.0010029765 0.02732353 0.0006071231
## 117   117 0.03214216 0.2089784 0.02450449 0.0010099631 0.02741933 0.0006062346
## 118   118 0.03214132 0.2090122 0.02450645 0.0010080833 0.02732887 0.0006033412
## 119   119 0.03214763 0.2087707 0.02451850 0.0010061729 0.02747402 0.0006028261
## 120   120 0.03215229 0.2085915 0.02451973 0.0010058761 0.02744672 0.0005998003
## 121   121 0.03215070 0.2086432 0.02450893 0.0010067038 0.02758981 0.0005978468
## 122   122 0.03214985 0.2087214 0.02450492 0.0010086638 0.02809764 0.0005995579
## 123   123 0.03216037 0.2082382 0.02451644 0.0010092376 0.02787817 0.0005959291
## 124   124 0.03216006 0.2082918 0.02451953 0.0010112696 0.02797384 0.0005950325
## 125   125 0.03215356 0.2085826 0.02451696 0.0010128226 0.02800961 0.0005928782
## 126   126 0.03215531 0.2085124 0.02451864 0.0010205569 0.02839401 0.0005909509
## 127   127 0.03215696 0.2084475 0.02451612 0.0010234017 0.02843550 0.0005855042
## 128   128 0.03215721 0.2084087 0.02451335 0.0010231192 0.02854995 0.0005874677
## 129   129 0.03215710 0.2084143 0.02450809 0.0010172609 0.02845232 0.0005768900
## 130   130 0.03215419 0.2085400 0.02450847 0.0010201942 0.02881985 0.0005738888
## 131   131 0.03215985 0.2082907 0.02451478 0.0010234146 0.02872738 0.0005743913
## 132   132 0.03215483 0.2085287 0.02450976 0.0010249697 0.02881318 0.0005766254
## 133   133 0.03215103 0.2087034 0.02450537 0.0010208831 0.02881511 0.0005777357
## 134   134 0.03215415 0.2085415 0.02450652 0.0010129106 0.02855833 0.0005693123
## 135   135 0.03215470 0.2085155 0.02450668 0.0010148192 0.02876493 0.0005673322
## 136   136 0.03215485 0.2085134 0.02450971 0.0010090371 0.02891441 0.0005693680
## 137   137 0.03215444 0.2085340 0.02450932 0.0010122713 0.02904098 0.0005774909
## 138   138 0.03216299 0.2082353 0.02451969 0.0010121684 0.02926101 0.0005738919
## 139   139 0.03216317 0.2082585 0.02451878 0.0010095397 0.02918148 0.0005616508
## 140   140 0.03215593 0.2085548 0.02451560 0.0010104557 0.02887843 0.0005622035
## 141   141 0.03216059 0.2083552 0.02451817 0.0010155237 0.02897989 0.0005675104
## 142   142 0.03215732 0.2085114 0.02451596 0.0010146523 0.02875212 0.0005686332
## 143   143 0.03214807 0.2089202 0.02451221 0.0010187396 0.02873564 0.0005754791
## 144   144 0.03214537 0.2090501 0.02450851 0.0010168166 0.02915286 0.0005713185
## 145   145 0.03214532 0.2090819 0.02450828 0.0010156859 0.02928343 0.0005659813
## 146   146 0.03214708 0.2090145 0.02451264 0.0010169684 0.02929652 0.0005705534
## 147   147 0.03214450 0.2091063 0.02450901 0.0010121788 0.02940405 0.0005709369
## 148   148 0.03214770 0.2089830 0.02451091 0.0010136493 0.02948067 0.0005661657
## 149   149 0.03214824 0.2089664 0.02451104 0.0010120980 0.02969934 0.0005632654
## 150   150 0.03214642 0.2090624 0.02451059 0.0010138204 0.02950451 0.0005645152
## 151   151 0.03214772 0.2090226 0.02451269 0.0010139140 0.02945682 0.0005639975
## 152   152 0.03214723 0.2090480 0.02451150 0.0010107555 0.02941016 0.0005610163
## 153   153 0.03215269 0.2088450 0.02452032 0.0010121873 0.02957134 0.0005601792
## 154   154 0.03215146 0.2088783 0.02452126 0.0010058699 0.02936871 0.0005548731
## 155   155 0.03215828 0.2085784 0.02452523 0.0010079364 0.02950134 0.0005569422
## 156   156 0.03215921 0.2085549 0.02452684 0.0010107211 0.02977898 0.0005553565
## 157   157 0.03215649 0.2086667 0.02452672 0.0010126726 0.02980571 0.0005574440
## 158   158 0.03216143 0.2084467 0.02452772 0.0010133377 0.02979671 0.0005559725
## 159   159 0.03216001 0.2085172 0.02452951 0.0010093986 0.02965062 0.0005547431
## 160   160 0.03216284 0.2084238 0.02453192 0.0010122031 0.02982704 0.0005587980
## 161   161 0.03216441 0.2083313 0.02453364 0.0010098008 0.02954716 0.0005591446
## 162   162 0.03216620 0.2082711 0.02453522 0.0010138113 0.02960062 0.0005580618
## 163   163 0.03217214 0.2080129 0.02454040 0.0010138947 0.02948098 0.0005587977
## 164   164 0.03216528 0.2083218 0.02453415 0.0010175377 0.02963785 0.0005587427
## 165   165 0.03216835 0.2081973 0.02453342 0.0010204827 0.02960225 0.0005601732
## 166   166 0.03217564 0.2078783 0.02453851 0.0010203277 0.02966775 0.0005591922
## 167   167 0.03217758 0.2077996 0.02454012 0.0010217168 0.02972841 0.0005597264
## 168   168 0.03217655 0.2078662 0.02453904 0.0010227459 0.02990458 0.0005590182
## 169   169 0.03217512 0.2079112 0.02453797 0.0010230722 0.02982936 0.0005610705
## 170   170 0.03217155 0.2080698 0.02453730 0.0010218811 0.02986280 0.0005654078
## 171   171 0.03216713 0.2082805 0.02453729 0.0010271596 0.03000340 0.0005722932
## 172   172 0.03216828 0.2082455 0.02453838 0.0010298547 0.03014195 0.0005759320
## 173   173 0.03216847 0.2082629 0.02453716 0.0010299276 0.03036903 0.0005812339
## 174   174 0.03216732 0.2083146 0.02453757 0.0010281120 0.03033647 0.0005786037
## 175   175 0.03216704 0.2083343 0.02453770 0.0010285093 0.03038131 0.0005789415
## 176   176 0.03216868 0.2082755 0.02453848 0.0010304159 0.03048129 0.0005849674
## 177   177 0.03216782 0.2083052 0.02453851 0.0010277548 0.03031289 0.0005831657
## 178   178 0.03216566 0.2084006 0.02453583 0.0010324762 0.03036976 0.0005873383
## 179   179 0.03216337 0.2084914 0.02453731 0.0010271580 0.03029283 0.0005844817
## 180   180 0.03216469 0.2084384 0.02453998 0.0010244198 0.03025427 0.0005810496
## 181   181 0.03216517 0.2084292 0.02454057 0.0010251342 0.03016464 0.0005794545
## 182   182 0.03216763 0.2083122 0.02454119 0.0010261439 0.03015605 0.0005794628
## 183   183 0.03216546 0.2084035 0.02454015 0.0010260660 0.03011890 0.0005780176
## 184   184 0.03216677 0.2083645 0.02454193 0.0010265081 0.03014826 0.0005760609
## 185   185 0.03216572 0.2084195 0.02454225 0.0010270003 0.03015033 0.0005770248
## 186   186 0.03216500 0.2084476 0.02454124 0.0010274774 0.03025564 0.0005803807
## 187   187 0.03216573 0.2084127 0.02454145 0.0010277030 0.03034973 0.0005819211
## 188   188 0.03216425 0.2084776 0.02454150 0.0010269486 0.03025515 0.0005810169
## 189   189 0.03216251 0.2085643 0.02454159 0.0010279261 0.03031291 0.0005806772
## 190   190 0.03216498 0.2084614 0.02454262 0.0010283071 0.03019995 0.0005804499
## 191   191 0.03216725 0.2083690 0.02454624 0.0010285162 0.03033460 0.0005774009
## 192   192 0.03216777 0.2083549 0.02454702 0.0010275493 0.03029612 0.0005743814
## 193   193 0.03216539 0.2084595 0.02454401 0.0010264475 0.03032567 0.0005755856
## 194   194 0.03216731 0.2083770 0.02454523 0.0010246960 0.03028185 0.0005737162
## 195   195 0.03216814 0.2083518 0.02454319 0.0010274000 0.03037978 0.0005758928
## 196   196 0.03216864 0.2083348 0.02454342 0.0010287042 0.03052692 0.0005770350
## 197   197 0.03217122 0.2082300 0.02454565 0.0010284786 0.03053989 0.0005776344
## 198   198 0.03217145 0.2082379 0.02454676 0.0010259938 0.03059299 0.0005779396
## 199   199 0.03217342 0.2081582 0.02454818 0.0010255665 0.03056133 0.0005769123
## 200   200 0.03217585 0.2080516 0.02455009 0.0010300977 0.03068622 0.0005812288
## 201   201 0.03217606 0.2080379 0.02454791 0.0010288608 0.03060724 0.0005817872
## 202   202 0.03217820 0.2079308 0.02454999 0.0010285276 0.03054389 0.0005817193
## 203   203 0.03217753 0.2079499 0.02454845 0.0010269300 0.03049128 0.0005818541
## 204   204 0.03217532 0.2080440 0.02454660 0.0010270832 0.03048089 0.0005805572
## 205   205 0.03217469 0.2080755 0.02454738 0.0010259199 0.03045193 0.0005817102
## 206   206 0.03217462 0.2080799 0.02454850 0.0010257996 0.03042095 0.0005828544
## 207   207 0.03217507 0.2080627 0.02454856 0.0010258137 0.03043646 0.0005833224
## 208   208 0.03217339 0.2081285 0.02454650 0.0010247221 0.03041088 0.0005815855
## 209   209 0.03217165 0.2082108 0.02454504 0.0010255701 0.03043851 0.0005799474
## 210   210 0.03217107 0.2082305 0.02454499 0.0010243892 0.03035694 0.0005794967
## 211   211 0.03217028 0.2082658 0.02454446 0.0010241793 0.03042870 0.0005784288
## 212   212 0.03217076 0.2082493 0.02454458 0.0010251336 0.03046675 0.0005799206
## 213   213 0.03217179 0.2081999 0.02454699 0.0010263623 0.03040213 0.0005819263
## 214   214 0.03217233 0.2081746 0.02454858 0.0010253640 0.03035722 0.0005829553
## 215   215 0.03217249 0.2081590 0.02454939 0.0010251119 0.03030147 0.0005819797
## 216   216 0.03217320 0.2081315 0.02455028 0.0010251587 0.03032362 0.0005814000
## 217   217 0.03217257 0.2081536 0.02454909 0.0010244459 0.03031425 0.0005802366
## 218   218 0.03217201 0.2081830 0.02454764 0.0010248497 0.03037021 0.0005800192
## 219   219 0.03217156 0.2082094 0.02454753 0.0010249053 0.03038442 0.0005804412
## 220   220 0.03217287 0.2081509 0.02454830 0.0010238949 0.03036826 0.0005798529
## 221   221 0.03217208 0.2081933 0.02454880 0.0010236598 0.03045166 0.0005788420
## 222   222 0.03217251 0.2081752 0.02454963 0.0010237822 0.03046285 0.0005791166
## 223   223 0.03217313 0.2081448 0.02455001 0.0010237917 0.03042657 0.0005790041
## 224   224 0.03217255 0.2081718 0.02454905 0.0010245804 0.03044162 0.0005805143
## 225   225 0.03217252 0.2081768 0.02454870 0.0010244913 0.03043456 0.0005804976
## 226   226 0.03217266 0.2081690 0.02454876 0.0010233782 0.03041217 0.0005797115
## 227   227 0.03217271 0.2081647 0.02454899 0.0010222782 0.03036312 0.0005788213
## 228   228 0.03217239 0.2081789 0.02454869 0.0010223895 0.03034406 0.0005789190
## 229   229 0.03217211 0.2081900 0.02454797 0.0010219123 0.03033818 0.0005791249
## 230   230 0.03217143 0.2082172 0.02454712 0.0010213524 0.03032561 0.0005787834
## 231   231 0.03217083 0.2082413 0.02454663 0.0010219359 0.03035036 0.0005789015
## 232   232 0.03217087 0.2082381 0.02454652 0.0010216273 0.03035559 0.0005790099
## 233   233 0.03217076 0.2082445 0.02454637 0.0010214425 0.03035973 0.0005790560
## 234   234 0.03217070 0.2082467 0.02454654 0.0010215979 0.03035273 0.0005787637
## 235   235 0.03217062 0.2082501 0.02454655 0.0010217329 0.03036635 0.0005786982
## 236   236 0.03217070 0.2082467 0.02454656 0.0010218717 0.03036421 0.0005786842
## 237   237 0.03217072 0.2082455 0.02454663 0.0010219775 0.03036520 0.0005787497
## 238   238 0.03217068 0.2082474 0.02454664 0.0010220484 0.03036455 0.0005787392
## 239   239 0.03217070 0.2082470 0.02454663 0.0010219997 0.03036450 0.0005787430
## 240   240 0.03217070 0.2082469 0.02454662 0.0010220164 0.03036344 0.0005787824
## [1] "Best Model"
##    nvmax
## 10    10

## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

## [1] "Coefficients of final model:"
##                  Estimate         2.5 %        97.5 %
## (Intercept)  1.998273e+00  1.9917085286  2.004838e+00
## x4          -5.278274e-05 -0.0000701811 -3.538438e-05
## x7           1.111579e-02  0.0098875519  1.234402e-02
## x9           3.312648e-03  0.0026744967  3.950798e-03
## x10          1.097724e-03  0.0005042726  1.691175e-03
## x16          9.043039e-04  0.0004909551  1.317653e-03
## x17          1.401563e-03  0.0007762335  2.026893e-03
## stat23       7.648038e-04  0.0002862667  1.243341e-03
## stat98       3.626398e-03  0.0031550681  4.097728e-03
## stat110     -3.209726e-03 -0.0036849776 -2.734474e-03
## x18.sqrt     2.651106e-02  0.0246828804  2.833923e-02

Test

if (algo.forward.caret == TRUE){
    test.model(model=model.forward, test=data.test
             ,method = 'leapForward',subopt = NULL
             ,formula = formula, feature.names = feature.names, label.names = label.names
             ,id = id
             ,draw.limits = TRUE, transformation = t)
}
## [1] "Summary of predicted values: "
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   2.041   2.085   2.097   2.097   2.110   2.148 
## [1] "leapForward  Test MSE: 0.00102809288031067"

Backward Elimination with CV

Train

if (algo.backward.caret == TRUE){
  set.seed(1)
  returned = train.caret.glmselect(formula = formula
                                   ,data =  data.train
                                   ,method = "leapBackward"
                                   ,feature.names =  feature.names)
  model.backward = returned$model
  id = returned$id
}
## Aggregating results
## Selecting tuning parameters
## Fitting nvmax = 10 on full training set
## [1] "All models results"
##     nvmax       RMSE  Rsquared        MAE       RMSESD RsquaredSD        MAESD
## 1       1 0.03394945 0.1136866 0.02639879 0.0009254924 0.02842538 0.0007668152
## 2       2 0.03313811 0.1544642 0.02563570 0.0008856904 0.02102133 0.0006503708
## 3       3 0.03252158 0.1856417 0.02506403 0.0009775242 0.02462482 0.0007440002
## 4       4 0.03203850 0.2095422 0.02440380 0.0010257703 0.02350731 0.0006318352
## 5       5 0.03175204 0.2236513 0.02420630 0.0010473102 0.02660189 0.0006818957
## 6       6 0.03165743 0.2282201 0.02412261 0.0009917289 0.02542266 0.0006188139
## 7       7 0.03167257 0.2274306 0.02415804 0.0009624469 0.02634536 0.0006081113
## 8       8 0.03160157 0.2308579 0.02409588 0.0009524258 0.02838942 0.0005958236
## 9       9 0.03157712 0.2319660 0.02408496 0.0009391885 0.02778037 0.0005830139
## 10     10 0.03156674 0.2325072 0.02407735 0.0009401267 0.02752636 0.0005792342
## 11     11 0.03157638 0.2320532 0.02409075 0.0009407004 0.02688228 0.0005850813
## 12     12 0.03159046 0.2314010 0.02412861 0.0009277078 0.02697054 0.0005820300
## 13     13 0.03157870 0.2319915 0.02411271 0.0009526699 0.02639659 0.0006047196
## 14     14 0.03158038 0.2319175 0.02409986 0.0009579561 0.02687828 0.0006030872
## 15     15 0.03161145 0.2305226 0.02412651 0.0009462155 0.02846025 0.0005743454
## 16     16 0.03161994 0.2301452 0.02413616 0.0009373097 0.02771921 0.0005727069
## 17     17 0.03163999 0.2293369 0.02415037 0.0009425634 0.02859923 0.0005749537
## 18     18 0.03165544 0.2286557 0.02417040 0.0009514967 0.02893632 0.0005773888
## 19     19 0.03165650 0.2286248 0.02415491 0.0009545003 0.02910990 0.0005945514
## 20     20 0.03166320 0.2283477 0.02415044 0.0009851954 0.02949750 0.0006249483
## 21     21 0.03168726 0.2272260 0.02415514 0.0009863788 0.02892454 0.0006253886
## 22     22 0.03171744 0.2259002 0.02418630 0.0010001762 0.02935038 0.0006335664
## 23     23 0.03172340 0.2257117 0.02418462 0.0010000441 0.02916448 0.0006635805
## 24     24 0.03173965 0.2249319 0.02418776 0.0009924356 0.02810294 0.0006619780
## 25     25 0.03172655 0.2255845 0.02418056 0.0009885731 0.02802019 0.0006523452
## 26     26 0.03174047 0.2249775 0.02419749 0.0009878392 0.02838716 0.0006507076
## 27     27 0.03176797 0.2237345 0.02422666 0.0009974540 0.02933490 0.0006717966
## 28     28 0.03178099 0.2231753 0.02423720 0.0010019531 0.02910119 0.0006795406
## 29     29 0.03178039 0.2232258 0.02424354 0.0009808457 0.02795681 0.0006621075
## 30     30 0.03178774 0.2228683 0.02424842 0.0009747523 0.02768208 0.0006640375
## 31     31 0.03178830 0.2228257 0.02424708 0.0009755222 0.02717124 0.0006707461
## 32     32 0.03178883 0.2228224 0.02423740 0.0009863400 0.02751103 0.0006701563
## 33     33 0.03180009 0.2222672 0.02424178 0.0009884028 0.02676024 0.0006816639
## 34     34 0.03181214 0.2217556 0.02426162 0.0009940650 0.02650230 0.0006842376
## 35     35 0.03182532 0.2211765 0.02427110 0.0009860586 0.02611359 0.0006714320
## 36     36 0.03184196 0.2204833 0.02428834 0.0009872704 0.02642778 0.0006701194
## 37     37 0.03186216 0.2195866 0.02430516 0.0009779149 0.02677591 0.0006598304
## 38     38 0.03186617 0.2194434 0.02430790 0.0009951061 0.02690108 0.0006665234
## 39     39 0.03188065 0.2188201 0.02431536 0.0009852954 0.02657996 0.0006625213
## 40     40 0.03190254 0.2178592 0.02433168 0.0009989136 0.02615765 0.0006594127
## 41     41 0.03190909 0.2175667 0.02432948 0.0010124971 0.02606341 0.0006580752
## 42     42 0.03189434 0.2183192 0.02430936 0.0010069528 0.02663517 0.0006589928
## 43     43 0.03189677 0.2181935 0.02430910 0.0009985134 0.02655181 0.0006523407
## 44     44 0.03190907 0.2176926 0.02431336 0.0010071927 0.02654015 0.0006629454
## 45     45 0.03190484 0.2178828 0.02431631 0.0010086549 0.02672135 0.0006714115
## 46     46 0.03191307 0.2175130 0.02433007 0.0010016322 0.02658507 0.0006582433
## 47     47 0.03191908 0.2172667 0.02433507 0.0010096167 0.02659260 0.0006629607
## 48     48 0.03193575 0.2165720 0.02435263 0.0010041966 0.02679596 0.0006651746
## 49     49 0.03193144 0.2168090 0.02434101 0.0010084842 0.02693422 0.0006650921
## 50     50 0.03194425 0.2163252 0.02434749 0.0010096890 0.02687145 0.0006687993
## 51     51 0.03193568 0.2167460 0.02434248 0.0010054255 0.02717544 0.0006596913
## 52     52 0.03193485 0.2168088 0.02434795 0.0009940003 0.02788691 0.0006520777
## 53     53 0.03194644 0.2163667 0.02435689 0.0009864323 0.02791905 0.0006412422
## 54     54 0.03195334 0.2160652 0.02436320 0.0009814347 0.02786441 0.0006319952
## 55     55 0.03196704 0.2154907 0.02437227 0.0009934246 0.02772135 0.0006437772
## 56     56 0.03197415 0.2152102 0.02437402 0.0009841789 0.02746529 0.0006242447
## 57     57 0.03197120 0.2153945 0.02436405 0.0009990305 0.02762770 0.0006474583
## 58     58 0.03197603 0.2151575 0.02436366 0.0009957173 0.02726344 0.0006470975
## 59     59 0.03198268 0.2148982 0.02437016 0.0009951252 0.02718796 0.0006445431
## 60     60 0.03198989 0.2145825 0.02438617 0.0009887287 0.02734812 0.0006503056
## 61     61 0.03199341 0.2144548 0.02438323 0.0009946833 0.02728599 0.0006471096
## 62     62 0.03200862 0.2138002 0.02438605 0.0009955816 0.02725365 0.0006487009
## 63     63 0.03200841 0.2138090 0.02438209 0.0009907199 0.02727704 0.0006436043
## 64     64 0.03202300 0.2131329 0.02439689 0.0009852217 0.02714042 0.0006380029
## 65     65 0.03202306 0.2131598 0.02440759 0.0009919660 0.02728313 0.0006321530
## 66     66 0.03202072 0.2133216 0.02440704 0.0009877456 0.02717308 0.0006366419
## 67     67 0.03201810 0.2134678 0.02440373 0.0009912814 0.02703649 0.0006363317
## 68     68 0.03201471 0.2136322 0.02439769 0.0009799462 0.02708470 0.0006345279
## 69     69 0.03202309 0.2132813 0.02440396 0.0009824140 0.02739904 0.0006405737
## 70     70 0.03201724 0.2135837 0.02440042 0.0009948132 0.02747461 0.0006496524
## 71     71 0.03201373 0.2137968 0.02439143 0.0010072786 0.02771347 0.0006595487
## 72     72 0.03201736 0.2136618 0.02439364 0.0010184789 0.02813343 0.0006594863
## 73     73 0.03201617 0.2136960 0.02438845 0.0010185013 0.02796820 0.0006595676
## 74     74 0.03203143 0.2130505 0.02440454 0.0010130963 0.02767060 0.0006540511
## 75     75 0.03203309 0.2130120 0.02441061 0.0010109526 0.02766841 0.0006562257
## 76     76 0.03203829 0.2128169 0.02440531 0.0009995916 0.02758747 0.0006446506
## 77     77 0.03204357 0.2126157 0.02440309 0.0010023751 0.02769252 0.0006355442
## 78     78 0.03205141 0.2122946 0.02440536 0.0010009054 0.02751667 0.0006293745
## 79     79 0.03205129 0.2123652 0.02440740 0.0010114305 0.02762578 0.0006353295
## 80     80 0.03206558 0.2117460 0.02442078 0.0010022133 0.02733630 0.0006330730
## 81     81 0.03206886 0.2116266 0.02442431 0.0010062159 0.02708932 0.0006314978
## 82     82 0.03207074 0.2115819 0.02442153 0.0010059865 0.02698265 0.0006271776
## 83     83 0.03207090 0.2116228 0.02442069 0.0010023057 0.02705011 0.0006321356
## 84     84 0.03206934 0.2116778 0.02442721 0.0010017448 0.02675346 0.0006315728
## 85     85 0.03208655 0.2109809 0.02443660 0.0010048497 0.02660096 0.0006356951
## 86     86 0.03209518 0.2105857 0.02444789 0.0010093451 0.02690607 0.0006440718
## 87     87 0.03208845 0.2108903 0.02443899 0.0010116066 0.02686679 0.0006483023
## 88     88 0.03208678 0.2109664 0.02443932 0.0010086274 0.02683112 0.0006509908
## 89     89 0.03209621 0.2104969 0.02445557 0.0010054909 0.02646443 0.0006541468
## 90     90 0.03210913 0.2099482 0.02446260 0.0009997085 0.02653059 0.0006451420
## 91     91 0.03212026 0.2095006 0.02446934 0.0009975353 0.02678852 0.0006473604
## 92     92 0.03211941 0.2095418 0.02446365 0.0009936705 0.02691375 0.0006375349
## 93     93 0.03210983 0.2099544 0.02446067 0.0010024465 0.02717207 0.0006438406
## 94     94 0.03210819 0.2100652 0.02445822 0.0010015466 0.02697495 0.0006446479
## 95     95 0.03212198 0.2095004 0.02446655 0.0010000565 0.02730874 0.0006339283
## 96     96 0.03213863 0.2088033 0.02448300 0.0009999745 0.02758102 0.0006249335
## 97     97 0.03213588 0.2089747 0.02448417 0.0010011489 0.02745338 0.0006258152
## 98     98 0.03214389 0.2086480 0.02448968 0.0009981864 0.02731392 0.0006253005
## 99     99 0.03215383 0.2082448 0.02449468 0.0010027450 0.02721306 0.0006229852
## 100   100 0.03215394 0.2082843 0.02449633 0.0010061346 0.02730629 0.0006239433
## 101   101 0.03214393 0.2087008 0.02449067 0.0010062626 0.02735432 0.0006187290
## 102   102 0.03214738 0.2085773 0.02449945 0.0010034775 0.02742810 0.0006185566
## 103   103 0.03215064 0.2084635 0.02449840 0.0009920961 0.02719653 0.0006218204
## 104   104 0.03215104 0.2084761 0.02449922 0.0009967835 0.02763902 0.0006227956
## 105   105 0.03214547 0.2087192 0.02449314 0.0009974022 0.02754824 0.0006265921
## 106   106 0.03214507 0.2087671 0.02449399 0.0009904404 0.02768798 0.0006144536
## 107   107 0.03214909 0.2086179 0.02449680 0.0009967512 0.02802728 0.0006156351
## 108   108 0.03215474 0.2083839 0.02450091 0.0009979818 0.02795696 0.0006135597
## 109   109 0.03214722 0.2087216 0.02449624 0.0010063407 0.02813961 0.0006156413
## 110   110 0.03214277 0.2088811 0.02449574 0.0010085720 0.02824481 0.0006165110
## 111   111 0.03214711 0.2087360 0.02449715 0.0010097424 0.02815227 0.0006187001
## 112   112 0.03214705 0.2087484 0.02449709 0.0010055423 0.02806928 0.0006176229
## 113   113 0.03213768 0.2091599 0.02449226 0.0010068736 0.02806601 0.0006152270
## 114   114 0.03214088 0.2090633 0.02449906 0.0010061388 0.02797819 0.0006112090
## 115   115 0.03213611 0.2092644 0.02449388 0.0010052617 0.02806171 0.0006092572
## 116   116 0.03213271 0.2094140 0.02449718 0.0010060322 0.02779079 0.0006060439
## 117   117 0.03213434 0.2093175 0.02450721 0.0010052816 0.02748523 0.0006047079
## 118   118 0.03213741 0.2092052 0.02450791 0.0010076273 0.02744864 0.0006064347
## 119   119 0.03214620 0.2088348 0.02451620 0.0009997492 0.02730815 0.0006021744
## 120   120 0.03215008 0.2086638 0.02451114 0.0010006854 0.02730257 0.0005958458
## 121   121 0.03214502 0.2089174 0.02450741 0.0010040478 0.02755131 0.0005940199
## 122   122 0.03214112 0.2090881 0.02451076 0.0010064171 0.02779773 0.0005906422
## 123   123 0.03214684 0.2088557 0.02451844 0.0010080574 0.02795621 0.0005840881
## 124   124 0.03216022 0.2082795 0.02452845 0.0010026669 0.02765239 0.0005812050
## 125   125 0.03216672 0.2080237 0.02453161 0.0010036223 0.02798998 0.0005800729
## 126   126 0.03216499 0.2080992 0.02452548 0.0010107632 0.02841649 0.0005835094
## 127   127 0.03216857 0.2079389 0.02452362 0.0010156000 0.02840952 0.0005849046
## 128   128 0.03216442 0.2081031 0.02451831 0.0010198915 0.02879110 0.0005850019
## 129   129 0.03216380 0.2081386 0.02451658 0.0010124916 0.02868471 0.0005771871
## 130   130 0.03216105 0.2082505 0.02451410 0.0010132807 0.02886890 0.0005749791
## 131   131 0.03215802 0.2083698 0.02451423 0.0010153817 0.02880625 0.0005721146
## 132   132 0.03215141 0.2086478 0.02451448 0.0010119863 0.02876489 0.0005711101
## 133   133 0.03215191 0.2086281 0.02451247 0.0010075110 0.02867369 0.0005677261
## 134   134 0.03215044 0.2086858 0.02450936 0.0009984016 0.02833800 0.0005618260
## 135   135 0.03215315 0.2085846 0.02451050 0.0010023579 0.02860006 0.0005621382
## 136   136 0.03215647 0.2084692 0.02451314 0.0009999775 0.02886836 0.0005547234
## 137   137 0.03215937 0.2083417 0.02451420 0.0010059135 0.02894965 0.0005614615
## 138   138 0.03216441 0.2081750 0.02452082 0.0010077139 0.02919008 0.0005618434
## 139   139 0.03216162 0.2083156 0.02452129 0.0010098126 0.02920175 0.0005588920
## 140   140 0.03215431 0.2086068 0.02451468 0.0010114175 0.02889777 0.0005622852
## 141   141 0.03215389 0.2086480 0.02451552 0.0010201850 0.02915818 0.0005696143
## 142   142 0.03214994 0.2088083 0.02451323 0.0010151935 0.02894520 0.0005692683
## 143   143 0.03214557 0.2090072 0.02450956 0.0010174738 0.02878613 0.0005758914
## 144   144 0.03214198 0.2091858 0.02450845 0.0010148946 0.02909032 0.0005705746
## 145   145 0.03214231 0.2091899 0.02450912 0.0010115108 0.02921405 0.0005660661
## 146   146 0.03214388 0.2091244 0.02451239 0.0010112613 0.02924717 0.0005686051
## 147   147 0.03214391 0.2091121 0.02450977 0.0010090029 0.02915773 0.0005700514
## 148   148 0.03214369 0.2091299 0.02451016 0.0010084516 0.02904924 0.0005617768
## 149   149 0.03214598 0.2090630 0.02451152 0.0010091806 0.02946608 0.0005604831
## 150   150 0.03214575 0.2090894 0.02451195 0.0010117383 0.02947525 0.0005624948
## 151   151 0.03214714 0.2090453 0.02451345 0.0010124894 0.02938864 0.0005638390
## 152   152 0.03214307 0.2092289 0.02451063 0.0010129354 0.02937239 0.0005621149
## 153   153 0.03215323 0.2088283 0.02452088 0.0010166190 0.02965298 0.0005606324
## 154   154 0.03215244 0.2088386 0.02452349 0.0010122437 0.02954604 0.0005560468
## 155   155 0.03215933 0.2085510 0.02452661 0.0010088598 0.02955574 0.0005576753
## 156   156 0.03215999 0.2085106 0.02452837 0.0010095093 0.02979502 0.0005549913
## 157   157 0.03215603 0.2086937 0.02452697 0.0010114274 0.02985036 0.0005572864
## 158   158 0.03215706 0.2086332 0.02452648 0.0010142263 0.02988366 0.0005560436
## 159   159 0.03215452 0.2087346 0.02452834 0.0010127590 0.02967141 0.0005554376
## 160   160 0.03215759 0.2085928 0.02452967 0.0010131428 0.02964639 0.0005576968
## 161   161 0.03216148 0.2084099 0.02453207 0.0010110858 0.02946024 0.0005573272
## 162   162 0.03216460 0.2083269 0.02453506 0.0010193498 0.02976422 0.0005603928
## 163   163 0.03216635 0.2082656 0.02453619 0.0010207782 0.02971550 0.0005589753
## 164   164 0.03216735 0.2082204 0.02453592 0.0010141319 0.02973671 0.0005551164
## 165   165 0.03216919 0.2081416 0.02453778 0.0010206955 0.02962173 0.0005575556
## 166   166 0.03216988 0.2081352 0.02453630 0.0010226704 0.02967263 0.0005619846
## 167   167 0.03217513 0.2079201 0.02454011 0.0010148784 0.02961592 0.0005587663
## 168   168 0.03217275 0.2080123 0.02453694 0.0010150199 0.02959222 0.0005603382
## 169   169 0.03217165 0.2080568 0.02453667 0.0010200135 0.02969491 0.0005643357
## 170   170 0.03217232 0.2080389 0.02453836 0.0010212802 0.02988860 0.0005642158
## 171   171 0.03217150 0.2081011 0.02453833 0.0010213726 0.02997863 0.0005692149
## 172   172 0.03217148 0.2081012 0.02453790 0.0010221357 0.02995818 0.0005712221
## 173   173 0.03216945 0.2082094 0.02453862 0.0010232913 0.03008610 0.0005751978
## 174   174 0.03217079 0.2081600 0.02454219 0.0010259523 0.03019784 0.0005755277
## 175   175 0.03216927 0.2082520 0.02453836 0.0010293551 0.03040440 0.0005803691
## 176   176 0.03216782 0.2083223 0.02453687 0.0010285968 0.03044002 0.0005828950
## 177   177 0.03216345 0.2084996 0.02453405 0.0010293186 0.03043702 0.0005826949
## 178   178 0.03216273 0.2085243 0.02453328 0.0010298796 0.03034763 0.0005846670
## 179   179 0.03216196 0.2085458 0.02453397 0.0010216004 0.03020790 0.0005796552
## 180   180 0.03216461 0.2084525 0.02453832 0.0010209770 0.03020610 0.0005794222
## 181   181 0.03216613 0.2083860 0.02454040 0.0010236231 0.03015416 0.0005797399
## 182   182 0.03216763 0.2083122 0.02454119 0.0010261439 0.03015605 0.0005794628
## 183   183 0.03216578 0.2083876 0.02454134 0.0010259473 0.03009563 0.0005788871
## 184   184 0.03216630 0.2083784 0.02454140 0.0010268770 0.03010432 0.0005794738
## 185   185 0.03216457 0.2084612 0.02454023 0.0010278252 0.03014091 0.0005794217
## 186   186 0.03216440 0.2084723 0.02454061 0.0010279354 0.03025685 0.0005812647
## 187   187 0.03216573 0.2084127 0.02454145 0.0010277030 0.03034973 0.0005819211
## 188   188 0.03216469 0.2084606 0.02454137 0.0010269914 0.03025916 0.0005809049
## 189   189 0.03216110 0.2086263 0.02454000 0.0010284747 0.03030733 0.0005797330
## 190   190 0.03216335 0.2085347 0.02454272 0.0010303248 0.03028216 0.0005801264
## 191   191 0.03216656 0.2083966 0.02454658 0.0010291156 0.03034294 0.0005767567
## 192   192 0.03216777 0.2083549 0.02454702 0.0010275493 0.03029612 0.0005743814
## 193   193 0.03216539 0.2084595 0.02454401 0.0010264475 0.03032567 0.0005755856
## 194   194 0.03216731 0.2083770 0.02454523 0.0010246960 0.03028185 0.0005737162
## 195   195 0.03216836 0.2083391 0.02454371 0.0010270605 0.03037638 0.0005750085
## 196   196 0.03216886 0.2083219 0.02454395 0.0010283579 0.03052344 0.0005761382
## 197   197 0.03217122 0.2082300 0.02454565 0.0010284786 0.03053989 0.0005776344
## 198   198 0.03217098 0.2082553 0.02454488 0.0010251899 0.03057905 0.0005747943
## 199   199 0.03217399 0.2081342 0.02454745 0.0010265662 0.03058061 0.0005756901
## 200   200 0.03217615 0.2080373 0.02454943 0.0010306281 0.03069775 0.0005801265
## 201   201 0.03217637 0.2080234 0.02454724 0.0010293994 0.03061888 0.0005806622
## 202   202 0.03217820 0.2079308 0.02454999 0.0010285276 0.03054389 0.0005817193
## 203   203 0.03217757 0.2079541 0.02454890 0.0010277543 0.03056129 0.0005829156
## 204   204 0.03217634 0.2080088 0.02454826 0.0010296158 0.03058324 0.0005836355
## 205   205 0.03217492 0.2080678 0.02454809 0.0010264898 0.03047412 0.0005822605
## 206   206 0.03217462 0.2080799 0.02454850 0.0010257996 0.03042095 0.0005828544
## 207   207 0.03217507 0.2080627 0.02454856 0.0010258137 0.03043646 0.0005833224
## 208   208 0.03217339 0.2081281 0.02454662 0.0010247142 0.03041039 0.0005816467
## 209   209 0.03217213 0.2081901 0.02454559 0.0010259768 0.03041250 0.0005802507
## 210   210 0.03217110 0.2082283 0.02454555 0.0010244070 0.03035416 0.0005798065
## 211   211 0.03217083 0.2082411 0.02454577 0.0010246431 0.03039790 0.0005791613
## 212   212 0.03217134 0.2082223 0.02454593 0.0010256037 0.03043276 0.0005806893
## 213   213 0.03217125 0.2082260 0.02454703 0.0010266110 0.03043993 0.0005818843
## 214   214 0.03217236 0.2081728 0.02454889 0.0010254332 0.03034316 0.0005835669
## 215   215 0.03217219 0.2081734 0.02454899 0.0010252970 0.03031067 0.0005820613
## 216   216 0.03217320 0.2081315 0.02455028 0.0010251587 0.03032362 0.0005814000
## 217   217 0.03217257 0.2081536 0.02454909 0.0010244459 0.03031425 0.0005802366
## 218   218 0.03217201 0.2081830 0.02454764 0.0010248497 0.03037021 0.0005800192
## 219   219 0.03217192 0.2081937 0.02454775 0.0010246411 0.03038423 0.0005801463
## 220   220 0.03217341 0.2081272 0.02454868 0.0010234998 0.03036801 0.0005793344
## 221   221 0.03217261 0.2081719 0.02454905 0.0010232707 0.03045158 0.0005785020
## 222   222 0.03217251 0.2081752 0.02454963 0.0010237822 0.03046285 0.0005791166
## 223   223 0.03217313 0.2081448 0.02455001 0.0010237917 0.03042657 0.0005790041
## 224   224 0.03217255 0.2081718 0.02454905 0.0010245804 0.03044162 0.0005805143
## 225   225 0.03217252 0.2081768 0.02454870 0.0010244913 0.03043456 0.0005804976
## 226   226 0.03217266 0.2081692 0.02454862 0.0010233798 0.03041234 0.0005796058
## 227   227 0.03217255 0.2081723 0.02454896 0.0010223373 0.03037415 0.0005788006
## 228   228 0.03217239 0.2081789 0.02454869 0.0010223895 0.03034406 0.0005789190
## 229   229 0.03217211 0.2081900 0.02454797 0.0010219123 0.03033818 0.0005791249
## 230   230 0.03217143 0.2082172 0.02454712 0.0010213524 0.03032561 0.0005787834
## 231   231 0.03217083 0.2082413 0.02454663 0.0010219359 0.03035036 0.0005789015
## 232   232 0.03217087 0.2082381 0.02454652 0.0010216273 0.03035559 0.0005790099
## 233   233 0.03217076 0.2082445 0.02454637 0.0010214425 0.03035973 0.0005790560
## 234   234 0.03217070 0.2082467 0.02454654 0.0010215979 0.03035273 0.0005787637
## 235   235 0.03217062 0.2082501 0.02454655 0.0010217329 0.03036635 0.0005786982
## 236   236 0.03217070 0.2082467 0.02454656 0.0010218717 0.03036421 0.0005786842
## 237   237 0.03217072 0.2082455 0.02454663 0.0010219775 0.03036520 0.0005787497
## 238   238 0.03217068 0.2082474 0.02454664 0.0010220484 0.03036455 0.0005787392
## 239   239 0.03217070 0.2082470 0.02454663 0.0010219997 0.03036450 0.0005787430
## 240   240 0.03217070 0.2082469 0.02454662 0.0010220164 0.03036344 0.0005787824
## [1] "Best Model"
##    nvmax
## 10    10

## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

## [1] "Coefficients of final model:"
##                  Estimate         2.5 %        97.5 %
## (Intercept)  1.998273e+00  1.9917085286  2.004838e+00
## x4          -5.278274e-05 -0.0000701811 -3.538438e-05
## x7           1.111579e-02  0.0098875519  1.234402e-02
## x9           3.312648e-03  0.0026744967  3.950798e-03
## x10          1.097724e-03  0.0005042726  1.691175e-03
## x16          9.043039e-04  0.0004909551  1.317653e-03
## x17          1.401563e-03  0.0007762335  2.026893e-03
## stat23       7.648038e-04  0.0002862667  1.243341e-03
## stat98       3.626398e-03  0.0031550681  4.097728e-03
## stat110     -3.209726e-03 -0.0036849776 -2.734474e-03
## x18.sqrt     2.651106e-02  0.0246828804  2.833923e-02

Test

if (algo.backward.caret == TRUE){
  test.model(model.backward, data.test
             ,method = 'leapBackward',subopt = NULL
             ,formula = formula, feature.names = feature.names, label.names = label.names
             ,id = id
             ,draw.limits = TRUE, transformation = t)
}
## [1] "Summary of predicted values: "
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   2.041   2.085   2.097   2.097   2.110   2.148 
## [1] "leapBackward  Test MSE: 0.00102809288031067"

Stepwise Selection with CV

Train

if (algo.stepwise.caret == TRUE){
  set.seed(1)
  returned = train.caret.glmselect(formula = formula
                                   ,data =  data.train
                                   ,method = "leapSeq"
                                   ,feature.names = feature.names)
  model.stepwise = returned$model
  id = returned$id
}
## Aggregating results
## Selecting tuning parameters
## Fitting nvmax = 10 on full training set
## [1] "All models results"
##     nvmax       RMSE  Rsquared        MAE       RMSESD RsquaredSD        MAESD
## 1       1 0.03394945 0.1136866 0.02639879 0.0009254924 0.02842538 0.0007668152
## 2       2 0.03313811 0.1544642 0.02563570 0.0008856904 0.02102133 0.0006503708
## 3       3 0.03252158 0.1856417 0.02506403 0.0009775242 0.02462482 0.0007440002
## 4       4 0.03203850 0.2095422 0.02440380 0.0010257703 0.02350731 0.0006318352
## 5       5 0.03175204 0.2236513 0.02420630 0.0010473102 0.02660189 0.0006818957
## 6       6 0.03165743 0.2282201 0.02412261 0.0009917289 0.02542266 0.0006188139
## 7       7 0.03167257 0.2274306 0.02415804 0.0009624469 0.02634536 0.0006081113
## 8       8 0.03160157 0.2308579 0.02409588 0.0009524258 0.02838942 0.0005958236
## 9       9 0.03157712 0.2319660 0.02408496 0.0009391885 0.02778037 0.0005830139
## 10     10 0.03156674 0.2325072 0.02407735 0.0009401267 0.02752636 0.0005792342
## 11     11 0.03157638 0.2320532 0.02409075 0.0009407004 0.02688228 0.0005850813
## 12     12 0.03159046 0.2314010 0.02412861 0.0009277078 0.02697054 0.0005820300
## 13     13 0.03157870 0.2319915 0.02411271 0.0009526699 0.02639659 0.0006047196
## 14     14 0.03158038 0.2319175 0.02409986 0.0009579561 0.02687828 0.0006030872
## 15     15 0.03161145 0.2305226 0.02412651 0.0009462155 0.02846025 0.0005743454
## 16     16 0.03160880 0.2306841 0.02412201 0.0009355650 0.02774726 0.0005601936
## 17     17 0.03195715 0.2136069 0.02441707 0.0010731182 0.04390539 0.0008861153
## 18     18 0.03165123 0.2288606 0.02416582 0.0009516104 0.02917152 0.0005717604
## 19     19 0.03221797 0.1996208 0.02468658 0.0017627550 0.07778835 0.0013715808
## 20     20 0.03166056 0.2284943 0.02415060 0.0009848732 0.02954986 0.0006246196
## 21     21 0.03168586 0.2273039 0.02415083 0.0009859269 0.02887674 0.0006218981
## 22     22 0.03242864 0.1896782 0.02475100 0.0018767209 0.07843990 0.0016031892
## 23     23 0.03172340 0.2257117 0.02418462 0.0010000441 0.02916448 0.0006635805
## 24     24 0.03173965 0.2249319 0.02418776 0.0009924356 0.02810294 0.0006619780
## 25     25 0.03203954 0.2093230 0.02444203 0.0014735641 0.06342596 0.0011813647
## 26     26 0.03173453 0.2252649 0.02419614 0.0009859999 0.02840956 0.0006488600
## 27     27 0.03215034 0.2046070 0.02449948 0.0016180150 0.06085325 0.0013529031
## 28     28 0.03178884 0.2228687 0.02425192 0.0010116432 0.02977919 0.0006952755
## 29     29 0.03207739 0.2076792 0.02448679 0.0014483230 0.06218627 0.0011378363
## 30     30 0.03178297 0.2231130 0.02425080 0.0009812289 0.02811992 0.0006618713
## 31     31 0.03240532 0.1914318 0.02473750 0.0019089069 0.07558507 0.0013958673
## 32     32 0.03179262 0.2226874 0.02424434 0.0009879393 0.02761461 0.0006773273
## 33     33 0.03237715 0.1916017 0.02473292 0.0014889589 0.07465230 0.0012504292
## 34     34 0.03236264 0.1939281 0.02478511 0.0014199582 0.05761348 0.0011692593
## 35     35 0.03247252 0.1891658 0.02480209 0.0016562440 0.06061761 0.0012425378
## 36     36 0.03184875 0.2202247 0.02429112 0.0010016696 0.02639855 0.0006660127
## 37     37 0.03185703 0.2198802 0.02430686 0.0009953940 0.02704123 0.0006650982
## 38     38 0.03213777 0.2050815 0.02451796 0.0014513609 0.06105012 0.0011318413
## 39     39 0.03186544 0.2195434 0.02430456 0.0010021742 0.02742110 0.0006746943
## 40     40 0.03187462 0.2191450 0.02431039 0.0010154026 0.02717873 0.0006832232
## 41     41 0.03220108 0.2029748 0.02456849 0.0016532923 0.05583053 0.0011312065
## 42     42 0.03208858 0.2079654 0.02454740 0.0013530325 0.04883273 0.0009837011
## 43     43 0.03274430 0.1738172 0.02505484 0.0015371124 0.07631412 0.0010759455
## 44     44 0.03214818 0.2051620 0.02451050 0.0013892859 0.06355404 0.0010745810
## 45     45 0.03250008 0.1878807 0.02477677 0.0018995700 0.06898388 0.0014373377
## 46     46 0.03222429 0.2025078 0.02456546 0.0016534380 0.04857898 0.0011676939
## 47     47 0.03211112 0.2070706 0.02456089 0.0013359234 0.04914549 0.0009554060
## 48     48 0.03193619 0.2165823 0.02434606 0.0009899309 0.02668962 0.0006544273
## 49     49 0.03276578 0.1738365 0.02501710 0.0018343118 0.07931750 0.0013154691
## 50     50 0.03262396 0.1816324 0.02489422 0.0015027250 0.07029531 0.0013291033
## 51     51 0.03243774 0.1909864 0.02481650 0.0018332930 0.06686264 0.0012609267
## 52     52 0.03193867 0.2166011 0.02435066 0.0009983049 0.02726082 0.0006700685
## 53     53 0.03249654 0.1866131 0.02488367 0.0010276395 0.06896966 0.0009304113
## 54     54 0.03195142 0.2161213 0.02435931 0.0009974662 0.02773515 0.0006395956
## 55     55 0.03239176 0.1946751 0.02466283 0.0015956913 0.05771720 0.0012837649
## 56     56 0.03228707 0.1996772 0.02461181 0.0016354796 0.05626819 0.0010859329
## 57     57 0.03225504 0.2001362 0.02461673 0.0011211450 0.05337197 0.0006974409
## 58     58 0.03289713 0.1692242 0.02513272 0.0021201743 0.07237344 0.0016102445
## 59     59 0.03241179 0.1939146 0.02466665 0.0016000688 0.05763065 0.0012658959
## 60     60 0.03200011 0.2141243 0.02439409 0.0009860366 0.02683785 0.0006174513
## 61     61 0.03200302 0.2139993 0.02438698 0.0009772920 0.02610587 0.0006378712
## 62     62 0.03201138 0.2135968 0.02439240 0.0009744271 0.02606968 0.0006353346
## 63     63 0.03220753 0.2030649 0.02463847 0.0013342585 0.04965664 0.0009215393
## 64     64 0.03312854 0.1559325 0.02532258 0.0019098965 0.08502187 0.0014174329
## 65     65 0.03202714 0.2129936 0.02442166 0.0009830883 0.02701733 0.0006343222
## 66     66 0.03245129 0.1922924 0.02471028 0.0015601957 0.05627138 0.0012254102
## 67     67 0.03205069 0.2120379 0.02443350 0.0009745383 0.02718322 0.0006256001
## 68     68 0.03288787 0.1682657 0.02515886 0.0013200129 0.07497604 0.0010919189
## 69     69 0.03294806 0.1675023 0.02512405 0.0019227243 0.06335722 0.0014230748
## 70     70 0.03293299 0.1679449 0.02516864 0.0020867165 0.07043491 0.0015583854
## 71     71 0.03259998 0.1833694 0.02488546 0.0016368666 0.06941783 0.0011657871
## 72     72 0.03317608 0.1525274 0.02538121 0.0011130442 0.07540876 0.0010450412
## 73     73 0.03203715 0.2127781 0.02442306 0.0009911103 0.02716038 0.0006457869
## 74     74 0.03230108 0.1986719 0.02461971 0.0013758994 0.05688604 0.0010524986
## 75     75 0.03280287 0.1714753 0.02518555 0.0013956967 0.07487207 0.0009352641
## 76     76 0.03257778 0.1847845 0.02481980 0.0018408770 0.07219628 0.0013378847
## 77     77 0.03231307 0.1979177 0.02466369 0.0011489747 0.05306639 0.0007255479
## 78     78 0.03264484 0.1811889 0.02493945 0.0011401264 0.06050411 0.0010384942
## 79     79 0.03276951 0.1775259 0.02493292 0.0019565756 0.06238202 0.0015015179
## 80     80 0.03274988 0.1769154 0.02495953 0.0015838681 0.06865092 0.0013183575
## 81     81 0.03235858 0.1971632 0.02465286 0.0016475139 0.05702330 0.0011135885
## 82     82 0.03265827 0.1817354 0.02492772 0.0015199617 0.06036289 0.0011453135
## 83     83 0.03262518 0.1826781 0.02489155 0.0016833306 0.06955485 0.0011002573
## 84     84 0.03301361 0.1633295 0.02518088 0.0017124299 0.08148129 0.0014556193
## 85     85 0.03302907 0.1627972 0.02519397 0.0015990191 0.08040395 0.0014106060
## 86     86 0.03305989 0.1620896 0.02515571 0.0021009950 0.08023238 0.0016513426
## 87     87 0.03282249 0.1708407 0.02518471 0.0014258669 0.07528675 0.0009542083
## 88     88 0.03209740 0.2105214 0.02445381 0.0010069465 0.02691729 0.0006371528
## 89     89 0.03209098 0.2107642 0.02445471 0.0010027411 0.02671862 0.0006436375
## 90     90 0.03312419 0.1559880 0.02542890 0.0016859545 0.08192188 0.0011004165
## 91     91 0.03238517 0.1950199 0.02471489 0.0011175077 0.05374399 0.0008665605
## 92     92 0.03270970 0.1797114 0.02496479 0.0017586211 0.06648198 0.0013543915
## 93     93 0.03211183 0.2099346 0.02446369 0.0009948230 0.02658273 0.0006448034
## 94     94 0.03242901 0.1936542 0.02471703 0.0011318639 0.04595110 0.0009589403
## 95     95 0.03237991 0.1953154 0.02471180 0.0011127406 0.05327586 0.0008426933
## 96     96 0.03296484 0.1652938 0.02521162 0.0013663467 0.07647123 0.0012404450
## 97     97 0.03239611 0.1946237 0.02472693 0.0011155800 0.05340049 0.0008373591
## 98     98 0.03244253 0.1931815 0.02471853 0.0014178887 0.05881587 0.0010898691
## 99     99 0.03284101 0.1735606 0.02502775 0.0014323350 0.06599630 0.0012281265
## 100   100 0.03215232 0.2083348 0.02449271 0.0009970851 0.02709271 0.0006211328
## 101   101 0.03243104 0.1934229 0.02475354 0.0009277755 0.05134876 0.0007298827
## 102   102 0.03268484 0.1813385 0.02494931 0.0018452074 0.07640601 0.0013662743
## 103   103 0.03272720 0.1777705 0.02499863 0.0011995349 0.06170256 0.0009449890
## 104   104 0.03317379 0.1581132 0.02525771 0.0020813426 0.07330661 0.0016707282
## 105   105 0.03259759 0.1838370 0.02495905 0.0014140482 0.06514370 0.0008888000
## 106   106 0.03214925 0.2085986 0.02449435 0.0009985385 0.02736906 0.0006179823
## 107   107 0.03349929 0.1387178 0.02568253 0.0020237609 0.07565748 0.0014176536
## 108   108 0.03296224 0.1665767 0.02520201 0.0017256298 0.08365020 0.0013442696
## 109   109 0.03259340 0.1850519 0.02495953 0.0016212179 0.07309802 0.0012013287
## 110   110 0.03316998 0.1579048 0.02529071 0.0017988435 0.06817624 0.0014582050
## 111   111 0.03273881 0.1773787 0.02500853 0.0012057800 0.06249465 0.0009447416
## 112   112 0.03321589 0.1553297 0.02531336 0.0017692260 0.07461960 0.0015981742
## 113   113 0.03267216 0.1809819 0.02497126 0.0014315925 0.07392432 0.0010728540
## 114   114 0.03285838 0.1730890 0.02506132 0.0014518687 0.06701624 0.0012639600
## 115   115 0.03244371 0.1934407 0.02473200 0.0014294981 0.05932562 0.0011063373
## 116   116 0.03241960 0.1951428 0.02471659 0.0016614443 0.05766016 0.0011091287
## 117   117 0.03214594 0.2088334 0.02451011 0.0009997412 0.02737145 0.0005977786
## 118   118 0.03214640 0.2088289 0.02451380 0.0010014840 0.02741122 0.0006016674
## 119   119 0.03277007 0.1775318 0.02505865 0.0018122985 0.06847228 0.0013748549
## 120   120 0.03278850 0.1751517 0.02511796 0.0013946911 0.05408655 0.0009877353
## 121   121 0.03214362 0.2089365 0.02450057 0.0010146558 0.02779812 0.0006052611
## 122   122 0.03276902 0.1755149 0.02507328 0.0007911870 0.05108578 0.0006663513
## 123   123 0.03240791 0.1955524 0.02472531 0.0010823920 0.04102648 0.0008473814
## 124   124 0.03262253 0.1838101 0.02498013 0.0013185990 0.06217985 0.0011552379
## 125   125 0.03296947 0.1682340 0.02518861 0.0018164088 0.06409393 0.0014470050
## 126   126 0.03216094 0.2082594 0.02452316 0.0010084702 0.02834547 0.0005782295
## 127   127 0.03280849 0.1749694 0.02513594 0.0014942344 0.06220057 0.0011589697
## 128   128 0.03280810 0.1740981 0.02515076 0.0013572385 0.07115607 0.0012175098
## 129   129 0.03251384 0.1911091 0.02474891 0.0014001927 0.04666538 0.0010511569
## 130   130 0.03247710 0.1908066 0.02490819 0.0011273091 0.05127216 0.0007828345
## 131   131 0.03288270 0.1706276 0.02514111 0.0012575053 0.06104458 0.0010844520
## 132   132 0.03215661 0.2084480 0.02451341 0.0010209609 0.02888962 0.0005745590
## 133   133 0.03215371 0.2085666 0.02450914 0.0010131187 0.02876804 0.0005739041
## 134   134 0.03215415 0.2085362 0.02450545 0.0010066829 0.02847818 0.0005660421
## 135   135 0.03264866 0.1840875 0.02484726 0.0011956063 0.03964017 0.0009609854
## 136   136 0.03253681 0.1891416 0.02487726 0.0014875883 0.05439835 0.0010256502
## 137   137 0.03250987 0.1906008 0.02482651 0.0014233541 0.05162506 0.0009379367
## 138   138 0.03216621 0.2080804 0.02452181 0.0010062646 0.02903225 0.0005625894
## 139   139 0.03236715 0.1976952 0.02471878 0.0012108692 0.04570701 0.0009304374
## 140   140 0.03296153 0.1668554 0.02518129 0.0011436014 0.05163071 0.0009524714
## 141   141 0.03245375 0.1928530 0.02480998 0.0011401457 0.04279263 0.0007809697
## 142   142 0.03215498 0.2086125 0.02451570 0.0010122960 0.02886396 0.0005684407
## 143   143 0.03214716 0.2089455 0.02451313 0.0010161244 0.02877042 0.0005716226
## 144   144 0.03235483 0.1982338 0.02471678 0.0012264016 0.04632698 0.0009677904
## 145   145 0.03234653 0.1984225 0.02468153 0.0010461542 0.04394512 0.0005577114
## 146   146 0.03238021 0.1974336 0.02468529 0.0010509685 0.03844663 0.0007395159
## 147   147 0.03259866 0.1866813 0.02492096 0.0016594114 0.06030572 0.0012617364
## 148   148 0.03258776 0.1871080 0.02487515 0.0012973476 0.05845392 0.0010223485
## 149   149 0.03235085 0.1984531 0.02471587 0.0012181188 0.04635894 0.0009638257
## 150   150 0.03251569 0.1900010 0.02488359 0.0012827284 0.06065605 0.0009909280
## 151   151 0.03237030 0.1983093 0.02472018 0.0013486968 0.05700418 0.0009880992
## 152   152 0.03271826 0.1806039 0.02501761 0.0016442501 0.06872814 0.0011875400
## 153   153 0.03235768 0.1982519 0.02472604 0.0012225727 0.04638107 0.0009651164
## 154   154 0.03272014 0.1795280 0.02505437 0.0012791583 0.04890808 0.0008376119
## 155   155 0.03216038 0.2084863 0.02452817 0.0010072275 0.02954461 0.0005552166
## 156   156 0.03252458 0.1891472 0.02491136 0.0011745445 0.05202402 0.0009882661
## 157   157 0.03225047 0.2031762 0.02467966 0.0011709313 0.04021971 0.0007051415
## 158   158 0.03239627 0.1968313 0.02469653 0.0010406074 0.03791252 0.0007168066
## 159   159 0.03267554 0.1810039 0.02501299 0.0010790233 0.05502121 0.0008328366
## 160   160 0.03259086 0.1878711 0.02491366 0.0015822523 0.05716283 0.0011458538
## 161   161 0.03239914 0.1971034 0.02472921 0.0015332461 0.05077350 0.0010153723
## 162   162 0.03276243 0.1778925 0.02512191 0.0014078728 0.06770724 0.0011942437
## 163   163 0.03233024 0.1993287 0.02471045 0.0010043291 0.04067881 0.0006740114
## 164   164 0.03216489 0.2083376 0.02453326 0.0010167219 0.02986181 0.0005576659
## 165   165 0.03277962 0.1776924 0.02508073 0.0015962425 0.06382228 0.0011922332
## 166   166 0.03234122 0.1988835 0.02471708 0.0010081407 0.04096266 0.0006741444
## 167   167 0.03246102 0.1940006 0.02472412 0.0012799470 0.03984186 0.0009007028
## 168   168 0.03236226 0.1979061 0.02470224 0.0010525776 0.04389879 0.0005671133
## 169   169 0.03262890 0.1862427 0.02494523 0.0017591481 0.06858698 0.0012885029
## 170   170 0.03281948 0.1743591 0.02521929 0.0014878651 0.05257707 0.0010993088
## 171   171 0.03216723 0.2082735 0.02453726 0.0010270477 0.02999529 0.0005723092
## 172   172 0.03217334 0.2080321 0.02453947 0.0010233810 0.03009193 0.0005727994
## 173   173 0.03241122 0.1963515 0.02471872 0.0010531241 0.03852660 0.0007416827
## 174   174 0.03216801 0.2082798 0.02453942 0.0010262472 0.03027234 0.0005775727
## 175   175 0.03233856 0.1991262 0.02472217 0.0010199003 0.04180416 0.0006917158
## 176   176 0.03269035 0.1812838 0.02500612 0.0012249120 0.04518785 0.0007477079
## 177   177 0.03237910 0.1975056 0.02473954 0.0012339832 0.04686500 0.0009606552
## 178   178 0.03257487 0.1876438 0.02490306 0.0014450851 0.04945524 0.0010391138
## 179   179 0.03265091 0.1851362 0.02492833 0.0015695986 0.05534651 0.0011406253
## 180   180 0.03216461 0.2084525 0.02453832 0.0010209770 0.03020610 0.0005794222
## 181   181 0.03241363 0.1968384 0.02474864 0.0015898330 0.05265930 0.0010763317
## 182   182 0.03256989 0.1878059 0.02484987 0.0011747217 0.04215858 0.0008624453
## 183   183 0.03216546 0.2084035 0.02454015 0.0010260660 0.03011890 0.0005780176
## 184   184 0.03237893 0.1981960 0.02473164 0.0013694920 0.03424397 0.0009019076
## 185   185 0.03246145 0.1941130 0.02474347 0.0013096070 0.04143682 0.0009588096
## 186   186 0.03237143 0.1977594 0.02471217 0.0010757632 0.04503961 0.0005944643
## 187   187 0.03237202 0.1977332 0.02471243 0.0010766350 0.04512435 0.0005971822
## 188   188 0.03216425 0.2084776 0.02454150 0.0010269486 0.03025515 0.0005810169
## 189   189 0.03262777 0.1853232 0.02491575 0.0010661879 0.04900854 0.0007421345
## 190   190 0.03216531 0.2084488 0.02454273 0.0010283377 0.03020287 0.0005805358
## 191   191 0.03237807 0.1975995 0.02474621 0.0012450944 0.04740953 0.0009587423
## 192   192 0.03293080 0.1707595 0.02512404 0.0012199081 0.04966815 0.0009816036
## 193   193 0.03263889 0.1862391 0.02495507 0.0018289950 0.05381599 0.0012725376
## 194   194 0.03216731 0.2083770 0.02454523 0.0010246960 0.03028185 0.0005737162
## 195   195 0.03246102 0.1941887 0.02474278 0.0013009727 0.04116745 0.0009454009
## 196   196 0.03265014 0.1859023 0.02496870 0.0018484114 0.05449506 0.0013018167
## 197   197 0.03258370 0.1872426 0.02487392 0.0011621499 0.04267742 0.0008702458
## 198   198 0.03217145 0.2082379 0.02454676 0.0010259938 0.03059299 0.0005779396
## 199   199 0.03246686 0.1939539 0.02475050 0.0012977957 0.04118948 0.0009545799
## 200   200 0.03243679 0.1961280 0.02477105 0.0016358406 0.05432222 0.0011209806
## 201   201 0.03217637 0.2080234 0.02454724 0.0010293994 0.03061888 0.0005806622
## 202   202 0.03243677 0.1960934 0.02476754 0.0016284698 0.05400619 0.0011125531
## 203   203 0.03229818 0.2011061 0.02467453 0.0008962142 0.03576095 0.0004976349
## 204   204 0.03217428 0.2080885 0.02454607 0.0010262156 0.03053746 0.0005802801
## 205   205 0.03217492 0.2080678 0.02454809 0.0010264898 0.03047412 0.0005822605
## 206   206 0.03280282 0.1778321 0.02516270 0.0019697145 0.06120572 0.0013482487
## 207   207 0.03217507 0.2080627 0.02454856 0.0010258137 0.03043646 0.0005833224
## 208   208 0.03217339 0.2081285 0.02454650 0.0010247221 0.03041088 0.0005815855
## 209   209 0.03261985 0.1862366 0.02493510 0.0013696731 0.06524598 0.0009671941
## 210   210 0.03240364 0.1966087 0.02474263 0.0011119460 0.04757743 0.0006474361
## 211   211 0.03239399 0.1975618 0.02474533 0.0013923221 0.03542250 0.0009306263
## 212   212 0.03217076 0.2082493 0.02454458 0.0010251336 0.03046675 0.0005799206
## 213   213 0.03217239 0.2081731 0.02454798 0.0010268762 0.03036863 0.0005824834
## 214   214 0.03244590 0.1958349 0.02478008 0.0016643162 0.05507536 0.0011503806
## 215   215 0.03269493 0.1830310 0.02495091 0.0015706367 0.04175422 0.0011671371
## 216   216 0.03286792 0.1738202 0.02513433 0.0014596457 0.06379314 0.0011929552
## 217   217 0.03239554 0.1974782 0.02475263 0.0013910806 0.03528693 0.0009382273
## 218   218 0.03240440 0.1966044 0.02474964 0.0011120398 0.04748012 0.0006560954
## 219   219 0.03247705 0.1935075 0.02475668 0.0013236632 0.04243936 0.0009762955
## 220   220 0.03217287 0.2081509 0.02454830 0.0010238949 0.03036826 0.0005798529
## 221   221 0.03290509 0.1723290 0.02515603 0.0016412542 0.04905052 0.0013317482
## 222   222 0.03217251 0.2081752 0.02454963 0.0010237822 0.03046285 0.0005791166
## 223   223 0.03238851 0.1972163 0.02475665 0.0012464117 0.04767892 0.0009740953
## 224   224 0.03243414 0.1956187 0.02476102 0.0010666439 0.03942481 0.0007963086
## 225   225 0.03217252 0.2081768 0.02454870 0.0010244913 0.03043456 0.0005804976
## 226   226 0.03217266 0.2081690 0.02454876 0.0010233782 0.03041217 0.0005797115
## 227   227 0.03235223 0.1987075 0.02473999 0.0010292766 0.04290605 0.0007121104
## 228   228 0.03268531 0.1835955 0.02494672 0.0015175229 0.06010180 0.0011987219
## 229   229 0.03269236 0.1832034 0.02494794 0.0015639194 0.04100045 0.0011618349
## 230   230 0.03245688 0.1955483 0.02479061 0.0016917337 0.05581532 0.0011814292
## 231   231 0.03217083 0.2082413 0.02454663 0.0010219359 0.03035036 0.0005789015
## 232   232 0.03282360 0.1761166 0.02516377 0.0013010340 0.05845302 0.0008987377
## 233   233 0.03217076 0.2082445 0.02454637 0.0010214425 0.03035973 0.0005790560
## 234   234 0.03217070 0.2082467 0.02454654 0.0010215979 0.03035273 0.0005787637
## 235   235 0.03302413 0.1663532 0.02537111 0.0015784958 0.07065563 0.0013009499
## 236   236 0.03262066 0.1861810 0.02495686 0.0015012946 0.05173675 0.0010960233
## 237   237 0.03300538 0.1675970 0.02531248 0.0017288616 0.05906769 0.0012236865
## 238   238 0.03353907 0.1420375 0.02579653 0.0019568554 0.07607180 0.0015220769
## 239   239 0.03345770 0.1450711 0.02576744 0.0016198827 0.06520788 0.0012702191
## 240   240 0.03217070 0.2082469 0.02454662 0.0010220164 0.03036344 0.0005787824
## [1] "Best Model"
##    nvmax
## 10    10

## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

## [1] "Coefficients of final model:"
##                  Estimate         2.5 %        97.5 %
## (Intercept)  1.998273e+00  1.9917085286  2.004838e+00
## x4          -5.278274e-05 -0.0000701811 -3.538438e-05
## x7           1.111579e-02  0.0098875519  1.234402e-02
## x9           3.312648e-03  0.0026744967  3.950798e-03
## x10          1.097724e-03  0.0005042726  1.691175e-03
## x16          9.043039e-04  0.0004909551  1.317653e-03
## x17          1.401563e-03  0.0007762335  2.026893e-03
## stat23       7.648038e-04  0.0002862667  1.243341e-03
## stat98       3.626398e-03  0.0031550681  4.097728e-03
## stat110     -3.209726e-03 -0.0036849776 -2.734474e-03
## x18.sqrt     2.651106e-02  0.0246828804  2.833923e-02

Test

if (algo.stepwise.caret == TRUE){
  test.model(model.stepwise, data.test
             ,method = 'leapSeq',subopt = NULL
             ,formula = formula, feature.names = feature.names, label.names = label.names
             ,id = id
             ,draw.limits = TRUE, transformation = t)
  
}
## [1] "Summary of predicted values: "
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   2.041   2.085   2.097   2.097   2.110   2.148 
## [1] "leapSeq  Test MSE: 0.00102809288031067"

LASSO with CV

Train

if (algo.LASSO.caret == TRUE){
  set.seed(1)
  tune.grid= expand.grid(alpha = 1,lambda = 10^seq(from=-4,to=-2,length=100))
  returned = train.caret.glmselect(formula = formula
                                   ,data =  data.train
                                   ,method = "glmnet"
                                   ,subopt = 'LASSO'
                                   ,tune.grid = tune.grid
                                   ,feature.names = feature.names)
  model.LASSO.caret = returned$model
}
## Aggregating results
## Selecting tuning parameters
## Fitting alpha = 1, lambda = 0.000643 on full training set
## glmnet 
## 
## 5584 samples
##  240 predictor
## 
## No pre-processing
## Resampling: Cross-Validated (10 fold) 
## Summary of sample sizes: 5026, 5026, 5026, 5025, 5025, 5026, ... 
## Resampling results across tuning parameters:
## 
##   lambda        RMSE        Rsquared   MAE       
##   0.0001000000  0.03196375  0.2152841  0.02438338
##   0.0001047616  0.03195590  0.2155665  0.02437738
##   0.0001097499  0.03194785  0.2158573  0.02437125
##   0.0001149757  0.03193959  0.2161570  0.02436490
##   0.0001204504  0.03193115  0.2164653  0.02435848
##   0.0001261857  0.03192254  0.2167820  0.02435197
##   0.0001321941  0.03191369  0.2171102  0.02434546
##   0.0001384886  0.03190461  0.2174495  0.02433886
##   0.0001450829  0.03189515  0.2178080  0.02433197
##   0.0001519911  0.03188555  0.2181745  0.02432499
##   0.0001592283  0.03187557  0.2185598  0.02431785
##   0.0001668101  0.03186556  0.2189483  0.02431063
##   0.0001747528  0.03185574  0.2193306  0.02430359
##   0.0001830738  0.03184591  0.2197162  0.02429668
##   0.0001917910  0.03183567  0.2201244  0.02428968
##   0.0002009233  0.03182530  0.2205433  0.02428253
##   0.0002104904  0.03181467  0.2209793  0.02427535
##   0.0002205131  0.03180401  0.2214217  0.02426830
##   0.0002310130  0.03179326  0.2218751  0.02426122
##   0.0002420128  0.03178250  0.2223344  0.02425441
##   0.0002535364  0.03177159  0.2228073  0.02424740
##   0.0002656088  0.03176092  0.2232752  0.02424045
##   0.0002782559  0.03175060  0.2237335  0.02423387
##   0.0002915053  0.03174040  0.2241942  0.02422768
##   0.0003053856  0.03172979  0.2246850  0.02422183
##   0.0003199267  0.03171916  0.2251862  0.02421612
##   0.0003351603  0.03170793  0.2257286  0.02420982
##   0.0003511192  0.03169676  0.2262795  0.02420369
##   0.0003678380  0.03168538  0.2268551  0.02419721
##   0.0003853529  0.03167452  0.2274189  0.02419117
##   0.0004037017  0.03166460  0.2279528  0.02418623
##   0.0004229243  0.03165511  0.2284804  0.02418208
##   0.0004430621  0.03164624  0.2289916  0.02417828
##   0.0004641589  0.03163809  0.2294814  0.02417529
##   0.0004862602  0.03163037  0.2299595  0.02417296
##   0.0005094138  0.03162389  0.2303893  0.02417221
##   0.0005336699  0.03161939  0.2307321  0.02417325
##   0.0005590810  0.03161624  0.2310203  0.02417485
##   0.0005857021  0.03161302  0.2313262  0.02417624
##   0.0006135907  0.03161075  0.2315989  0.02417831
##   0.0006428073  0.03160969  0.2318219  0.02418141
##   0.0006734151  0.03161018  0.2319798  0.02418580
##   0.0007054802  0.03161239  0.2320639  0.02419178
##   0.0007390722  0.03161648  0.2320668  0.02419943
##   0.0007742637  0.03162207  0.2320069  0.02420857
##   0.0008111308  0.03162900  0.2318933  0.02421868
##   0.0008497534  0.03163702  0.2317451  0.02422945
##   0.0008902151  0.03164628  0.2315536  0.02424187
##   0.0009326033  0.03165819  0.2312449  0.02425602
##   0.0009770100  0.03167175  0.2308733  0.02427126
##   0.0010235310  0.03168677  0.2304501  0.02428751
##   0.0010722672  0.03170347  0.2299595  0.02430485
##   0.0011233240  0.03172054  0.2294823  0.02432271
##   0.0011768120  0.03173897  0.2289621  0.02434215
##   0.0012328467  0.03175806  0.2284445  0.02436283
##   0.0012915497  0.03177901  0.2278610  0.02438508
##   0.0013530478  0.03180140  0.2272412  0.02440840
##   0.0014174742  0.03182595  0.2265417  0.02443412
##   0.0014849683  0.03185201  0.2258057  0.02446155
##   0.0015556761  0.03187988  0.2250157  0.02449065
##   0.0016297508  0.03190658  0.2243538  0.02451975
##   0.0017073526  0.03193508  0.2236493  0.02455033
##   0.0017886495  0.03196425  0.2229793  0.02458169
##   0.0018738174  0.03199549  0.2222651  0.02461513
##   0.0019630407  0.03202754  0.2215930  0.02465048
##   0.0020565123  0.03206238  0.2208469  0.02468840
##   0.0021544347  0.03210051  0.2199979  0.02472852
##   0.0022570197  0.03214215  0.2190345  0.02477121
##   0.0023644894  0.03218562  0.2180679  0.02481486
##   0.0024770764  0.03223256  0.2169998  0.02486140
##   0.0025950242  0.03227989  0.2160620  0.02490803
##   0.0027185882  0.03233121  0.2150125  0.02495723
##   0.0028480359  0.03238704  0.2138266  0.02500947
##   0.0029836472  0.03244808  0.2124538  0.02506636
##   0.0031257158  0.03251494  0.2108499  0.02512792
##   0.0032745492  0.03258814  0.2089687  0.02519391
##   0.0034304693  0.03266828  0.2067533  0.02526548
##   0.0035938137  0.03275599  0.2041340  0.02534202
##   0.0037649358  0.03285197  0.2010246  0.02542505
##   0.0039442061  0.03295531  0.1974590  0.02551446
##   0.0041320124  0.03305949  0.1939540  0.02560496
##   0.0043287613  0.03316857  0.1901780  0.02569812
##   0.0045348785  0.03327812  0.1865996  0.02579131
##   0.0047508102  0.03339673  0.1823521  0.02588973
##   0.0049770236  0.03352608  0.1771917  0.02599661
##   0.0052140083  0.03366656  0.1709216  0.02611185
##   0.0054622772  0.03380582  0.1646528  0.02622367
##   0.0057223677  0.03394979  0.1576908  0.02633800
##   0.0059948425  0.03409014  0.1509617  0.02644838
##   0.0062802914  0.03423859  0.1430321  0.02656353
##   0.0065793322  0.03437705  0.1358868  0.02666926
##   0.0068926121  0.03451674  0.1281852  0.02677402
##   0.0072208090  0.03464066  0.1221849  0.02686676
##   0.0075646333  0.03476249  0.1160630  0.02695792
##   0.0079248290  0.03485443  0.1144871  0.02702542
##   0.0083021757  0.03494662  0.1136942  0.02709246
##   0.0086974900  0.03504246  0.1136866  0.02716242
##   0.0091116276  0.03514732  0.1136866  0.02723900
##   0.0095454846  0.03526204  0.1136866  0.02732250
##   0.0100000000  0.03538753  0.1136866  0.02741351
## 
## Tuning parameter 'alpha' was held constant at a value of 1
## RMSE was used to select the optimal model using the smallest value.
## The final values used for the model were alpha = 1 and lambda = 0.0006428073.

##    alpha       lambda
## 41     1 0.0006428073
##     alpha       lambda       RMSE  Rsquared        MAE       RMSESD RsquaredSD        MAESD
## 1       1 0.0001000000 0.03196375 0.2152841 0.02438338 0.0010081705 0.02908048 0.0005824678
## 2       1 0.0001047616 0.03195590 0.2155665 0.02437738 0.0010073724 0.02902574 0.0005826690
## 3       1 0.0001097499 0.03194785 0.2158573 0.02437125 0.0010064108 0.02896109 0.0005829199
## 4       1 0.0001149757 0.03193959 0.2161570 0.02436490 0.0010054031 0.02889272 0.0005833180
## 5       1 0.0001204504 0.03193115 0.2164653 0.02435848 0.0010044008 0.02882480 0.0005839344
## 6       1 0.0001261857 0.03192254 0.2167820 0.02435197 0.0010034642 0.02875519 0.0005847040
## 7       1 0.0001321941 0.03191369 0.2171102 0.02434546 0.0010024838 0.02868595 0.0005854578
## 8       1 0.0001384886 0.03190461 0.2174495 0.02433886 0.0010014291 0.02861315 0.0005861909
## 9       1 0.0001450829 0.03189515 0.2178080 0.02433197 0.0010002769 0.02854351 0.0005870573
## 10      1 0.0001519911 0.03188555 0.2181745 0.02432499 0.0009991501 0.02847248 0.0005881338
## 11      1 0.0001592283 0.03187557 0.2185598 0.02431785 0.0009979670 0.02840621 0.0005891694
## 12      1 0.0001668101 0.03186556 0.2189483 0.02431063 0.0009968877 0.02833702 0.0005903079
## 13      1 0.0001747528 0.03185574 0.2193306 0.02430359 0.0009957842 0.02825664 0.0005916803
## 14      1 0.0001830738 0.03184591 0.2197162 0.02429668 0.0009945701 0.02817266 0.0005928800
## 15      1 0.0001917910 0.03183567 0.2201244 0.02428968 0.0009934861 0.02809418 0.0005938098
## 16      1 0.0002009233 0.03182530 0.2205433 0.02428253 0.0009925914 0.02801364 0.0005949065
## 17      1 0.0002104904 0.03181467 0.2209793 0.02427535 0.0009919370 0.02793757 0.0005964299
## 18      1 0.0002205131 0.03180401 0.2214217 0.02426830 0.0009915431 0.02786024 0.0005983986
## 19      1 0.0002310130 0.03179326 0.2218751 0.02426122 0.0009913120 0.02778938 0.0006005144
## 20      1 0.0002420128 0.03178250 0.2223344 0.02425441 0.0009910361 0.02771469 0.0006030714
## 21      1 0.0002535364 0.03177159 0.2228073 0.02424740 0.0009908185 0.02766071 0.0006057971
## 22      1 0.0002656088 0.03176092 0.2232752 0.02424045 0.0009903319 0.02760529 0.0006085495
## 23      1 0.0002782559 0.03175060 0.2237335 0.02423387 0.0009895127 0.02754061 0.0006110543
## 24      1 0.0002915053 0.03174040 0.2241942 0.02422768 0.0009886858 0.02747428 0.0006134181
## 25      1 0.0003053856 0.03172979 0.2246850 0.02422183 0.0009879302 0.02742249 0.0006150551
## 26      1 0.0003199267 0.03171916 0.2251862 0.02421612 0.0009870169 0.02736318 0.0006167210
## 27      1 0.0003351603 0.03170793 0.2257286 0.02420982 0.0009865257 0.02734481 0.0006181919
## 28      1 0.0003511192 0.03169676 0.2262795 0.02420369 0.0009860162 0.02733672 0.0006197106
## 29      1 0.0003678380 0.03168538 0.2268551 0.02419721 0.0009850188 0.02735800 0.0006209477
## 30      1 0.0003853529 0.03167452 0.2274189 0.02419117 0.0009842277 0.02739197 0.0006222189
## 31      1 0.0004037017 0.03166460 0.2279528 0.02418623 0.0009832174 0.02743959 0.0006226711
## 32      1 0.0004229243 0.03165511 0.2284804 0.02418208 0.0009822089 0.02749407 0.0006230472
## 33      1 0.0004430621 0.03164624 0.2289916 0.02417828 0.0009805474 0.02751746 0.0006222338
## 34      1 0.0004641589 0.03163809 0.2294814 0.02417529 0.0009793186 0.02754325 0.0006213322
## 35      1 0.0004862602 0.03163037 0.2299595 0.02417296 0.0009776289 0.02753221 0.0006202337
## 36      1 0.0005094138 0.03162389 0.2303893 0.02417221 0.0009765328 0.02751681 0.0006186742
## 37      1 0.0005336699 0.03161939 0.2307321 0.02417325 0.0009752079 0.02752462 0.0006170905
## 38      1 0.0005590810 0.03161624 0.2310203 0.02417485 0.0009735474 0.02752708 0.0006153099
## 39      1 0.0005857021 0.03161302 0.2313262 0.02417624 0.0009725681 0.02753069 0.0006137869
## 40      1 0.0006135907 0.03161075 0.2315989 0.02417831 0.0009717410 0.02752171 0.0006124450
## 41      1 0.0006428073 0.03160969 0.2318219 0.02418141 0.0009705768 0.02748745 0.0006105894
## 42      1 0.0006734151 0.03161018 0.2319798 0.02418580 0.0009695254 0.02745551 0.0006089519
## 43      1 0.0007054802 0.03161239 0.2320639 0.02419178 0.0009678804 0.02737972 0.0006064324
## 44      1 0.0007390722 0.03161648 0.2320668 0.02419943 0.0009670016 0.02729768 0.0006041964
## 45      1 0.0007742637 0.03162207 0.2320069 0.02420857 0.0009658836 0.02717049 0.0006009225
## 46      1 0.0008111308 0.03162900 0.2318933 0.02421868 0.0009648447 0.02704294 0.0005976386
## 47      1 0.0008497534 0.03163702 0.2317451 0.02422945 0.0009632625 0.02691643 0.0005946942
## 48      1 0.0008902151 0.03164628 0.2315536 0.02424187 0.0009619528 0.02676386 0.0005922121
## 49      1 0.0009326033 0.03165819 0.2312449 0.02425602 0.0009618590 0.02663623 0.0005921917
## 50      1 0.0009770100 0.03167175 0.2308733 0.02427126 0.0009621348 0.02648762 0.0005922922
## 51      1 0.0010235310 0.03168677 0.2304501 0.02428751 0.0009625322 0.02634647 0.0005925872
## 52      1 0.0010722672 0.03170347 0.2299595 0.02430485 0.0009627651 0.02620382 0.0005928509
## 53      1 0.0011233240 0.03172054 0.2294823 0.02432271 0.0009630284 0.02608427 0.0005921693
## 54      1 0.0011768120 0.03173897 0.2289621 0.02434215 0.0009643609 0.02595758 0.0005919292
## 55      1 0.0012328467 0.03175806 0.2284445 0.02436283 0.0009662957 0.02587959 0.0005920794
## 56      1 0.0012915497 0.03177901 0.2278610 0.02438508 0.0009682875 0.02579744 0.0005922821
## 57      1 0.0013530478 0.03180140 0.2272412 0.02440840 0.0009698671 0.02570600 0.0005927495
## 58      1 0.0014174742 0.03182595 0.2265417 0.02443412 0.0009711866 0.02559356 0.0005932865
## 59      1 0.0014849683 0.03185201 0.2258057 0.02446155 0.0009718432 0.02544671 0.0005938525
## 60      1 0.0015556761 0.03187988 0.2250157 0.02449065 0.0009725413 0.02527176 0.0005945474
## 61      1 0.0016297508 0.03190658 0.2243538 0.02451975 0.0009729124 0.02515917 0.0005949531
## 62      1 0.0017073526 0.03193508 0.2236493 0.02455033 0.0009726384 0.02502117 0.0005953806
## 63      1 0.0017886495 0.03196425 0.2229793 0.02458169 0.0009733610 0.02494960 0.0005961808
## 64      1 0.0018738174 0.03199549 0.2222651 0.02461513 0.0009743642 0.02488525 0.0005971256
## 65      1 0.0019630407 0.03202754 0.2215930 0.02465048 0.0009756310 0.02482892 0.0005983040
## 66      1 0.0020565123 0.03206238 0.2208469 0.02468840 0.0009768874 0.02475725 0.0005998659
## 67      1 0.0021544347 0.03210051 0.2199979 0.02472852 0.0009781363 0.02467895 0.0006016750
## 68      1 0.0022570197 0.03214215 0.2190345 0.02477121 0.0009791937 0.02459462 0.0006037610
## 69      1 0.0023644894 0.03218562 0.2180679 0.02481486 0.0009784581 0.02436996 0.0006039544
## 70      1 0.0024770764 0.03223256 0.2169998 0.02486140 0.0009762605 0.02415123 0.0006041129
## 71      1 0.0025950242 0.03227989 0.2160620 0.02490803 0.0009740056 0.02408688 0.0006040594
## 72      1 0.0027185882 0.03233121 0.2150125 0.02495723 0.0009717118 0.02400533 0.0006049895
## 73      1 0.0028480359 0.03238704 0.2138266 0.02500947 0.0009696819 0.02392179 0.0006069954
## 74      1 0.0029836472 0.03244808 0.2124538 0.02506636 0.0009678994 0.02384638 0.0006097908
## 75      1 0.0031257158 0.03251494 0.2108499 0.02512792 0.0009662730 0.02378577 0.0006131918
## 76      1 0.0032745492 0.03258814 0.2089687 0.02519391 0.0009648410 0.02374600 0.0006171180
## 77      1 0.0034304693 0.03266828 0.2067533 0.02526548 0.0009636474 0.02373471 0.0006220794
## 78      1 0.0035938137 0.03275599 0.2041340 0.02534202 0.0009627425 0.02376141 0.0006272548
## 79      1 0.0037649358 0.03285197 0.2010246 0.02542505 0.0009621834 0.02383765 0.0006336586
## 80      1 0.0039442061 0.03295531 0.1974590 0.02551446 0.0009609493 0.02404345 0.0006393890
## 81      1 0.0041320124 0.03305949 0.1939540 0.02560496 0.0009618546 0.02418256 0.0006440509
## 82      1 0.0043287613 0.03316857 0.1901780 0.02569812 0.0009639588 0.02434074 0.0006502379
## 83      1 0.0045348785 0.03327812 0.1865996 0.02579131 0.0009645193 0.02510077 0.0006566687
## 84      1 0.0047508102 0.03339673 0.1823521 0.02588973 0.0009654847 0.02587128 0.0006636738
## 85      1 0.0049770236 0.03352608 0.1771917 0.02599661 0.0009671359 0.02678037 0.0006706072
## 86      1 0.0052140083 0.03366656 0.1709216 0.02611185 0.0009688497 0.02785489 0.0006750826
## 87      1 0.0054622772 0.03380582 0.1646528 0.02622367 0.0009726210 0.02854744 0.0006755480
## 88      1 0.0057223677 0.03394979 0.1576908 0.02633800 0.0009777009 0.02876464 0.0006746510
## 89      1 0.0059948425 0.03409014 0.1509617 0.02644838 0.0009800266 0.02926649 0.0006670055
## 90      1 0.0062802914 0.03423859 0.1430321 0.02656353 0.0009822869 0.02976516 0.0006604192
## 91      1 0.0065793322 0.03437705 0.1358868 0.02666926 0.0009872377 0.03026349 0.0006558360
## 92      1 0.0068926121 0.03451674 0.1281852 0.02677402 0.0009929318 0.03022809 0.0006532267
## 93      1 0.0072208090 0.03464066 0.1221849 0.02686676 0.0009976870 0.03009116 0.0006492397
## 94      1 0.0075646333 0.03476249 0.1160630 0.02695792 0.0010011431 0.02927961 0.0006453485
## 95      1 0.0079248290 0.03485443 0.1144871 0.02702542 0.0010058202 0.02913544 0.0006410281
## 96      1 0.0083021757 0.03494662 0.1136942 0.02709246 0.0010132020 0.02842502 0.0006379724
## 97      1 0.0086974900 0.03504246 0.1136866 0.02716242 0.0010205705 0.02842538 0.0006360778
## 98      1 0.0091116276 0.03514732 0.1136866 0.02723900 0.0010285349 0.02842538 0.0006359131
## 99      1 0.0095454846 0.03526204 0.1136866 0.02732250 0.0010370922 0.02842538 0.0006365861
## 100     1 0.0100000000 0.03538753 0.1136866 0.02741351 0.0010462734 0.02842538 0.0006380887

## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

## [1] "Coefficients"
##                model.coef
## (Intercept)  2.000152e+00
## x4          -3.889499e-05
## x7           1.008553e-02
## x8           1.650980e-04
## x9           2.789522e-03
## x10          6.388010e-04
## x11          6.149195e+04
## x16          5.457325e-04
## x17          9.436512e-04
## x21          6.181878e-05
## x22         -1.157496e-04
## stat4       -2.062319e-04
## stat5       -5.272020e-05
## stat6       -6.812657e-06
## stat13      -2.905926e-04
## stat14      -3.276968e-04
## stat18      -2.107958e-06
## stat22      -2.234160e-05
## stat23       3.505555e-04
## stat24      -7.424774e-05
## stat30       5.038789e-05
## stat35      -6.184752e-05
## stat39      -4.525321e-05
## stat41      -1.172574e-04
## stat45      -6.814191e-05
## stat51       1.238986e-04
## stat59       1.198847e-04
## stat91      -1.768373e-04
## stat92      -1.251910e-06
## stat98       3.259828e-03
## stat100      1.585566e-04
## stat106     -6.951533e-05
## stat110     -2.851618e-03
## stat113     -1.061505e-04
## stat131      9.366626e-06
## stat146     -1.016484e-04
## stat149     -2.155991e-05
## stat156      1.345530e-05
## stat170     -1.050902e-05
## stat175     -1.004477e-04
## stat189      1.012390e-05
## stat195      7.687884e-05
## stat204     -2.449476e-05
## stat207      1.142682e-04
## x18.sqrt     2.505718e-02

Test

if (algo.LASSO.caret == TRUE){
  test.model(model.LASSO.caret, data.test
             ,method = 'glmnet',subopt = "LASSO"
             ,formula = formula, feature.names = feature.names, label.names = label.names
             ,draw.limits = TRUE, transformation = t)
}
## [1] "Summary of predicted values: "
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   2.043   2.086   2.097   2.097   2.109   2.142 
## [1] "glmnet LASSO Test MSE: 0.00102582818356386"

LARS with CV

Train

if (algo.LARS.caret == TRUE){
  set.seed(1)
  returned = train.caret.glmselect(formula = formula
                                   ,data =  data.train
                                   ,method = "lars"
                                   ,subopt = 'NULL'
                                   ,feature.names = feature.names)
  model.LARS.caret = returned$model
}
## Warning in nominalTrainWorkflow(x = x, y = y, wts = weights, info = trainInfo, : There were missing values in resampled
## performance measures.
## Aggregating results
## Selecting tuning parameters
## Fitting fraction = 0.343 on full training set
## Least Angle Regression 
## 
## 5584 samples
##  240 predictor
## 
## Pre-processing: centered (240), scaled (240) 
## Resampling: Cross-Validated (10 fold) 
## Summary of sample sizes: 5026, 5026, 5026, 5025, 5025, 5026, ... 
## Resampling results across tuning parameters:
## 
##   fraction    RMSE        Rsquared   MAE       
##   0.00000000  0.03601297        NaN  0.02786690
##   0.01010101  0.03559181  0.1136866  0.02756153
##   0.02020202  0.03521559  0.1136866  0.02728803
##   0.03030303  0.03488574  0.1136866  0.02704866
##   0.04040404  0.03461069  0.1233874  0.02684348
##   0.05050505  0.03435218  0.1363429  0.02664979
##   0.06060606  0.03410970  0.1500367  0.02646227
##   0.07070707  0.03388808  0.1604771  0.02628947
##   0.08080808  0.03367963  0.1704554  0.02612252
##   0.09090909  0.03347922  0.1793248  0.02595811
##   0.10101010  0.03329104  0.1863149  0.02580240
##   0.11111111  0.03311700  0.1919679  0.02565428
##   0.12121212  0.03295574  0.1976123  0.02551525
##   0.13131313  0.03280184  0.2028686  0.02538294
##   0.14141414  0.03265764  0.2072276  0.02525683
##   0.15151515  0.03252365  0.2107876  0.02513604
##   0.16161616  0.03239999  0.2136822  0.02502145
##   0.17171717  0.03228716  0.2160152  0.02491422
##   0.18181818  0.03218722  0.2181457  0.02481689
##   0.19191919  0.03209920  0.2199797  0.02472748
##   0.20202020  0.03201788  0.2217724  0.02464145
##   0.21212121  0.03194644  0.2233545  0.02456381
##   0.22222222  0.03188252  0.2250458  0.02449612
##   0.23232323  0.03182620  0.2265381  0.02443621
##   0.24242424  0.03177724  0.2278929  0.02438411
##   0.25252525  0.03173735  0.2289879  0.02434108
##   0.26262626  0.03170562  0.2298260  0.02430699
##   0.27272727  0.03167699  0.2306620  0.02427691
##   0.28282828  0.03165611  0.2312099  0.02425323
##   0.29292929  0.03163831  0.2316969  0.02423161
##   0.30303030  0.03162595  0.2319768  0.02421550
##   0.31313131  0.03161866  0.2320470  0.02420373
##   0.32323232  0.03161335  0.2320612  0.02419397
##   0.33333333  0.03161019  0.2320016  0.02418623
##   0.34343434  0.03160926  0.2318578  0.02418137
##   0.35353535  0.03161021  0.2316440  0.02417797
##   0.36363636  0.03161231  0.2313918  0.02417594
##   0.37373737  0.03161516  0.2311218  0.02417517
##   0.38383838  0.03161725  0.2309090  0.02417386
##   0.39393939  0.03162008  0.2306671  0.02417260
##   0.40404040  0.03162360  0.2303995  0.02417166
##   0.41414141  0.03162825  0.2300865  0.02417166
##   0.42424242  0.03163339  0.2297593  0.02417295
##   0.43434343  0.03163909  0.2294100  0.02417476
##   0.44444444  0.03164496  0.2290578  0.02417710
##   0.45454545  0.03165105  0.2287011  0.02417949
##   0.46464646  0.03165716  0.2283498  0.02418203
##   0.47474747  0.03166344  0.2279978  0.02418475
##   0.48484848  0.03167030  0.2276244  0.02418831
##   0.49494949  0.03167741  0.2272469  0.02419235
##   0.50505051  0.03168496  0.2268564  0.02419674
##   0.51515152  0.03169265  0.2264666  0.02420123
##   0.52525253  0.03170049  0.2260757  0.02420556
##   0.53535354  0.03170853  0.2256805  0.02420988
##   0.54545455  0.03171652  0.2252945  0.02421434
##   0.55555556  0.03172390  0.2249432  0.02421844
##   0.56565657  0.03173126  0.2245993  0.02422243
##   0.57575758  0.03173861  0.2242605  0.02422657
##   0.58585859  0.03174608  0.2239213  0.02423098
##   0.59595960  0.03175368  0.2235810  0.02423557
##   0.60606061  0.03176140  0.2232391  0.02424039
##   0.61616162  0.03176947  0.2228843  0.02424570
##   0.62626263  0.03177797  0.2225137  0.02425110
##   0.63636364  0.03178660  0.2221407  0.02425649
##   0.64646465  0.03179518  0.2217754  0.02426191
##   0.65656566  0.03180377  0.2214140  0.02426753
##   0.66666667  0.03181240  0.2210555  0.02427308
##   0.67676768  0.03182116  0.2206966  0.02427888
##   0.68686869  0.03183015  0.2203314  0.02428503
##   0.69696970  0.03183911  0.2199724  0.02429122
##   0.70707071  0.03184800  0.2196213  0.02429740
##   0.71717172  0.03185688  0.2192743  0.02430362
##   0.72727273  0.03186624  0.2189093  0.02431026
##   0.73737374  0.03187600  0.2185296  0.02431734
##   0.74747475  0.03188587  0.2181487  0.02432450
##   0.75757576  0.03189568  0.2177760  0.02433174
##   0.76767677  0.03190559  0.2174027  0.02433912
##   0.77777778  0.03191566  0.2170262  0.02434648
##   0.78787879  0.03192575  0.2166540  0.02435399
##   0.79797980  0.03193600  0.2162793  0.02436180
##   0.80808081  0.03194648  0.2158979  0.02436998
##   0.81818182  0.03195710  0.2155137  0.02437810
##   0.82828283  0.03196777  0.2151308  0.02438629
##   0.83838384  0.03197868  0.2147414  0.02439474
##   0.84848485  0.03198988  0.2143425  0.02440337
##   0.85858586  0.03200145  0.2139305  0.02441260
##   0.86868687  0.03201284  0.2135310  0.02442171
##   0.87878788  0.03202441  0.2131282  0.02443103
##   0.88888889  0.03203610  0.2127244  0.02444058
##   0.89898990  0.03204778  0.2123242  0.02445004
##   0.90909091  0.03205935  0.2119313  0.02445932
##   0.91919192  0.03207108  0.2115349  0.02446856
##   0.92929293  0.03208309  0.2111304  0.02447804
##   0.93939394  0.03209532  0.2107194  0.02448757
##   0.94949495  0.03210769  0.2103069  0.02449726
##   0.95959596  0.03212011  0.2098958  0.02450692
##   0.96969697  0.03213258  0.2094852  0.02451668
##   0.97979798  0.03214514  0.2090754  0.02452649
##   0.98989899  0.03215788  0.2086613  0.02453648
##   1.00000000  0.03217070  0.2082469  0.02454662
## 
## RMSE was used to select the optimal model using the smallest value.
## The final value used for the model was fraction = 0.3434343.

##     fraction
## 35 0.3434343
## Warning: Removed 1 rows containing missing values (geom_point).

## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.

## [1] "Coefficients"
##            x4            x7            x8            x9           x10           x11           x16           x17 
## -1.840779e-03  6.783491e-03  4.699723e-04  3.605551e-03  8.833224e-04  3.499088e-04  1.084712e-03  1.238578e-03 
##           x21           x22         stat4         stat5        stat13        stat14        stat22        stat23 
##  6.214232e-04 -1.370420e-04 -3.490437e-04 -8.514952e-05 -4.954764e-04 -5.631424e-04 -3.116292e-05  5.995927e-04 
##        stat24        stat30        stat35        stat39        stat41        stat45        stat51        stat59 
## -1.210631e-04  7.941615e-05 -9.981285e-05 -7.425737e-05 -1.977596e-04 -1.112973e-04  2.076700e-04  2.005853e-04 
##        stat91        stat98       stat100       stat106       stat110       stat113       stat131       stat146 
## -3.023325e-04  5.712654e-03  2.665406e-04 -1.143700e-04 -4.953476e-03 -1.768466e-04  2.103190e-06 -1.679350e-04 
##       stat149       stat156       stat175       stat195       stat204       stat207      x18.sqrt 
## -2.970753e-05  5.749412e-06 -1.662038e-04  1.254047e-04 -3.443971e-05  1.901566e-04  1.133440e-02

Test

if (algo.LARS.caret == TRUE){
  test.model(model.LARS.caret, data.test
             ,method = 'lars',subopt = NULL
             ,formula = formula, feature.names = feature.names, label.names = label.names
             ,draw.limits = TRUE, transformation = t)
}
## [1] "Summary of predicted values: "
##    Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
##   2.044   2.086   2.097   2.097   2.109   2.142 
## [1] "lars  Test MSE: 0.00102598369365549"

Session Info

sessionInfo()
## R version 3.5.2 (2018-12-20)
## Platform: x86_64-w64-mingw32/x64 (64-bit)
## Running under: Windows 10 x64 (build 17763)
## 
## Matrix products: default
## 
## locale:
## [1] LC_COLLATE=English_United States.1252  LC_CTYPE=English_United States.1252    LC_MONETARY=English_United States.1252
## [4] LC_NUMERIC=C                           LC_TIME=English_United States.1252    
## 
## attached base packages:
## [1] parallel  stats     graphics  grDevices utils     datasets  methods   base     
## 
## other attached packages:
##  [1] knitr_1.21                 htmltools_0.3.6            reshape2_1.4.3             lars_1.2                  
##  [5] doParallel_1.0.14          iterators_1.0.10           caret_6.0-81               leaps_3.0                 
##  [9] ggforce_0.1.3              rlist_0.4.6.1              car_3.0-2                  carData_3.0-2             
## [13] bestNormalize_1.3.0        scales_1.0.0               onewaytests_2.0            caTools_1.17.1.1          
## [17] mosaic_1.5.0               mosaicData_0.17.0          ggformula_0.9.1            ggstance_0.3.1            
## [21] lattice_0.20-38            DT_0.5                     ggiraphExtra_0.2.9         ggiraph_0.6.0             
## [25] investr_1.4.0              glmnet_2.0-16              foreach_1.4.4              Matrix_1.2-15             
## [29] MASS_7.3-51.1              PerformanceAnalytics_1.5.2 xts_0.11-2                 zoo_1.8-4                 
## [33] forcats_0.3.0              stringr_1.4.0              dplyr_0.8.0.1              purrr_0.3.0               
## [37] readr_1.3.1                tidyr_0.8.2                tibble_2.0.1               ggplot2_3.1.0             
## [41] tidyverse_1.2.1            usdm_1.1-18                raster_2.8-19              sp_1.3-1                  
## [45] pacman_0.5.0              
## 
## loaded via a namespace (and not attached):
##  [1] readxl_1.3.0       backports_1.1.3    plyr_1.8.4         lazyeval_0.2.1     splines_3.5.2      mycor_0.1.1       
##  [7] crosstalk_1.0.0    leaflet_2.0.2      digest_0.6.18      magrittr_1.5       mosaicCore_0.6.0   openxlsx_4.1.0    
## [13] recipes_0.1.4      modelr_0.1.3       gower_0.1.2        colorspace_1.4-0   rvest_0.3.2        ggrepel_0.8.0     
## [19] haven_2.0.0        xfun_0.4           crayon_1.3.4       jsonlite_1.6       survival_2.43-3    glue_1.3.0        
## [25] registry_0.5       gtable_0.2.0       ppcor_1.1          ipred_0.9-8        sjmisc_2.7.7       abind_1.4-5       
## [31] rngtools_1.3.1     bibtex_0.4.2       Rcpp_1.0.0         xtable_1.8-3       units_0.6-2        foreign_0.8-71    
## [37] stats4_3.5.2       lava_1.6.5         prodlim_2018.04.18 prediction_0.3.6.2 htmlwidgets_1.3    httr_1.4.0        
## [43] RColorBrewer_1.1-2 pkgconfig_2.0.2    farver_1.1.0       nnet_7.3-12        labeling_0.3       tidyselect_0.2.5  
## [49] rlang_0.3.1        later_0.8.0        munsell_0.5.0      cellranger_1.1.0   tools_3.5.2        cli_1.0.1         
## [55] generics_0.0.2     moments_0.14       sjlabelled_1.0.16  broom_0.5.1        evaluate_0.13      ggdendro_0.1-20   
## [61] yaml_2.2.0         ModelMetrics_1.2.2 zip_1.0.0          nlme_3.1-137       doRNG_1.7.1        mime_0.6          
## [67] xml2_1.2.0         compiler_3.5.2     rstudioapi_0.9.0   curl_3.3           tweenr_1.0.1       stringi_1.3.1     
## [73] highr_0.7          gdtools_0.1.7      stringdist_0.9.5.1 pillar_1.3.1       data.table_1.12.0  bitops_1.0-6      
## [79] httpuv_1.4.5.1     R6_2.4.0           promises_1.0.1     gridExtra_2.3      rio_0.5.16         codetools_0.2-15  
## [85] assertthat_0.2.0   pkgmaker_0.27      withr_2.1.2        nortest_1.0-4      mgcv_1.8-26        hms_0.4.2         
## [91] quadprog_1.5-5     grid_3.5.2         rpart_4.1-13       timeDate_3043.102  class_7.3-14       rmarkdown_1.11    
## [97] snakecase_0.9.2    shiny_1.2.0        lubridate_1.7.4